66 lines
2.6 KiB
Python
66 lines
2.6 KiB
Python
from django.core.management.base import BaseCommand
|
|
from core.models import Website
|
|
from core.utils import full_site_crawler
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "全站递归爬取 人民日报及其子网站、客户端、新媒体平台"
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('--platform', type=str, default='all',
|
|
choices=['peopleapp', 'people', 'paper', 'all'],
|
|
help='选择爬取平台: peopleapp(客户端), people(人民网), paper(报纸), all(全部)')
|
|
|
|
def handle(self, *args, **options):
|
|
platform = options['platform']
|
|
|
|
# 人民日报各平台配置
|
|
platforms = {
|
|
'peopleapp': {
|
|
'name': '人民日报客户端',
|
|
'base_url': 'https://www.peopleapp.com',
|
|
'start_url': 'https://www.peopleapp.com/home',
|
|
'article_selector': 'a'
|
|
},
|
|
'people': {
|
|
'name': '人民网',
|
|
'base_url': 'https://www.people.com.cn',
|
|
'start_url': 'https://www.people.com.cn',
|
|
'article_selector': 'a'
|
|
},
|
|
'paper': {
|
|
'name': '人民日报报纸',
|
|
'base_url': 'http://paper.people.com.cn',
|
|
'start_url': 'http://paper.people.com.cn',
|
|
'article_selector': 'a'
|
|
}
|
|
}
|
|
|
|
if platform == 'all':
|
|
target_platforms = platforms.values()
|
|
else:
|
|
target_platforms = [platforms[platform]]
|
|
|
|
for platform_config in target_platforms:
|
|
website, created = Website.objects.get_or_create(
|
|
name=platform_config['name'],
|
|
defaults={
|
|
'base_url': platform_config['base_url'],
|
|
'article_list_url': platform_config['start_url'],
|
|
'article_selector': platform_config['article_selector']
|
|
}
|
|
)
|
|
|
|
# 确保更新已存在的网站对象的配置
|
|
if not created:
|
|
website.base_url = platform_config['base_url']
|
|
website.article_list_url = platform_config['start_url']
|
|
website.article_selector = platform_config['article_selector']
|
|
website.save()
|
|
|
|
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
|
|
full_site_crawler(platform_config['start_url'], website, max_pages=500)
|
|
self.stdout.write(f"完成爬取: {platform_config['name']}")
|
|
|
|
self.stdout.write(self.style.SUCCESS("人民日报所有平台爬取完成"))
|