21 lines
756 B
Python
21 lines
756 B
Python
# core/management/commands/crawl_dongfangyancao.py
|
|
from django.core.management.base import BaseCommand
|
|
from core.models import Website
|
|
from core.utils import full_site_crawler
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "全站递归爬取 东方烟草报"
|
|
|
|
def handle(self, *args, **kwargs):
|
|
website, created = Website.objects.get_or_create(
|
|
name="东方烟草报",
|
|
defaults={
|
|
'article_list_url': 'https://www.eastobacco.com/',
|
|
'article_selector': 'a'
|
|
}
|
|
)
|
|
start_url = "https://www.eastobacco.com/"
|
|
self.stdout.write(f"开始全站爬取: {start_url}")
|
|
full_site_crawler(start_url, website, max_pages=500)
|
|
self.stdout.write("爬取完成") |