diff --git a/core/management/commands/crawl_cngov.py b/core/management/commands/crawl_cngov.py index 0bd09e9..4dbcb27 100644 --- a/core/management/commands/crawl_cngov.py +++ b/core/management/commands/crawl_cngov.py @@ -4,17 +4,50 @@ from core.utils import full_site_crawler class Command(BaseCommand): - help = "全站递归爬取 www.gov.cn" + help = "全站递归爬取 中国政府网及其子网站" - def handle(self, *args, **kwargs): - website, created = Website.objects.get_or_create( - name="中国政府网", - defaults={ - 'article_list_url': 'https://www.gov.cn/', + def add_arguments(self, parser): + parser.add_argument('--platform', type=str, default='all', + choices=['govcn', 'all'], + help='选择爬取平台: govcn(中国政府网), all(全部)') + + def handle(self, *args, **options): + platform = options['platform'] + + # 中国政府网各平台配置 + platforms = { + 'govcn': { + 'name': '中国政府网', + 'base_url': 'https://www.gov.cn/', + 'start_url': 'https://www.gov.cn/', 'article_selector': 'a' - } - ) - start_url = "https://www.gov.cn/" - self.stdout.write(f"开始全站爬取: {start_url}") - full_site_crawler(start_url, website, max_pages=500) - self.stdout.write("爬取完成") + }, + } + + if platform == 'all': + target_platforms = platforms.values() + else: + target_platforms = [platforms[platform]] + + for platform_config in target_platforms: + website, created = Website.objects.get_or_create( + name=platform_config['name'], + defaults={ + 'base_url': platform_config['base_url'], + 'article_list_url': platform_config['start_url'], + 'article_selector': platform_config['article_selector'] + } + ) + + # 确保更新已存在的网站对象的配置 + if not created: + website.base_url = platform_config['base_url'] + website.article_list_url = platform_config['start_url'] + website.article_selector = platform_config['article_selector'] + website.save() + + self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") + full_site_crawler(platform_config['start_url'], website, max_pages=500) + self.stdout.write(f"完成爬取: {platform_config['name']}") + + self.stdout.write(self.style.SUCCESS("中国政府网所有平台爬取完成")) \ No newline at end of file diff --git a/core/management/commands/crawl_dongfangyancao.py b/core/management/commands/crawl_dongfangyancao.py index 2f50a7d..b5561a9 100644 --- a/core/management/commands/crawl_dongfangyancao.py +++ b/core/management/commands/crawl_dongfangyancao.py @@ -6,15 +6,48 @@ from core.utils import full_site_crawler class Command(BaseCommand): help = "全站递归爬取 东方烟草报" - def handle(self, *args, **kwargs): - website, created = Website.objects.get_or_create( - name="东方烟草报", - defaults={ - 'article_list_url': 'https://www.eastobacco.com/', + def add_arguments(self, parser): + parser.add_argument('--platform', type=str, default='all', + choices=['eastobacco', 'all'], + help='选择爬取平台: eastobacco(东方烟草报), all(全部)') + + def handle(self, *args, **options): + platform = options['platform'] + + # 东方烟草报各平台配置 + platforms = { + 'eastobacco': { + 'name': '东方烟草报', + 'base_url': 'https://www.eastobacco.com/', + 'start_url': 'https://www.eastobacco.com/', 'article_selector': 'a' - } - ) - start_url = "https://www.eastobacco.com/" - self.stdout.write(f"开始全站爬取: {start_url}") - full_site_crawler(start_url, website, max_pages=500) - self.stdout.write("爬取完成") + }, + } + + if platform == 'all': + target_platforms = platforms.values() + else: + target_platforms = [platforms[platform]] + + for platform_config in target_platforms: + website, created = Website.objects.get_or_create( + name=platform_config['name'], + defaults={ + 'base_url': platform_config['base_url'], + 'article_list_url': platform_config['start_url'], + 'article_selector': platform_config['article_selector'] + } + ) + + # 确保更新已存在的网站对象的配置 + if not created: + website.base_url = platform_config['base_url'] + website.article_list_url = platform_config['start_url'] + website.article_selector = platform_config['article_selector'] + website.save() + + self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") + full_site_crawler(platform_config['start_url'], website, max_pages=500) + self.stdout.write(f"完成爬取: {platform_config['name']}") + + self.stdout.write(self.style.SUCCESS("东方烟草报所有平台爬取完成")) \ No newline at end of file