26 lines
994 B
Python
26 lines
994 B
Python
from django.core.management.base import BaseCommand
|
|
from core.models import Website
|
|
from core.utils import full_site_crawler
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "全站递归爬取 人民日报 https://www.peopleapp.com"
|
|
|
|
def handle(self, *args, **kwargs):
|
|
website, created = Website.objects.get_or_create(
|
|
name="人民日报",
|
|
defaults={
|
|
'article_list_url': 'https://www.peopleapp.com/home',
|
|
'article_selector': 'a',
|
|
'base_url': 'https://www.peopleapp.com'
|
|
}
|
|
)
|
|
# 确保更新已存在的网站对象的base_url
|
|
if not created and not website.base_url:
|
|
website.base_url = 'https://www.peopleapp.com'
|
|
website.save()
|
|
|
|
start_url = "https://www.peopleapp.com/home"
|
|
self.stdout.write(f"开始全站爬取: {start_url}")
|
|
full_site_crawler(start_url, website, max_pages=500)
|
|
self.stdout.write("爬取完成") |