Change cngov && dongfangyancao setup

core/management/commands/crawl_cngov.py

@@ -4,17 +4,50 @@ from core.utils import full_site_crawler
 
 
 class Command(BaseCommand):
-    help = "全站递归爬取 www.gov.cn"
+    help = "全站递归爬取 中国政府网及其子网站"
 
-    def handle(self, *args, **kwargs):
-        website, created = Website.objects.get_or_create(
-            name="中国政府网",
-            defaults={
-                'article_list_url': 'https://www.gov.cn/',
+    def add_arguments(self, parser):
+        parser.add_argument('--platform', type=str, default='all',
+                            choices=['govcn', 'all'],
+                            help='选择爬取平台: govcn(中国政府网), all(全部)')
+
+    def handle(self, *args, **options):
+        platform = options['platform']
+
+        # 中国政府网各平台配置
+        platforms = {
+            'govcn': {
+                'name': '中国政府网',
+                'base_url': 'https://www.gov.cn/',
+                'start_url': 'https://www.gov.cn/',
                 'article_selector': 'a'
-            }
-        )
-        start_url = "https://www.gov.cn/"
-        self.stdout.write(f"开始全站爬取: {start_url}")
-        full_site_crawler(start_url, website, max_pages=500)
-        self.stdout.write("爬取完成")
+            },
+        }
+
+        if platform == 'all':
+            target_platforms = platforms.values()
+        else:
+            target_platforms = [platforms[platform]]
+
+        for platform_config in target_platforms:
+            website, created = Website.objects.get_or_create(
+                name=platform_config['name'],
+                defaults={
+                    'base_url': platform_config['base_url'],
+                    'article_list_url': platform_config['start_url'],
+                    'article_selector': platform_config['article_selector']
+                }
+            )
+
+            # 确保更新已存在的网站对象的配置
+            if not created:
+                website.base_url = platform_config['base_url']
+                website.article_list_url = platform_config['start_url']
+                website.article_selector = platform_config['article_selector']
+                website.save()
+
+            self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
+            full_site_crawler(platform_config['start_url'], website, max_pages=500)
+            self.stdout.write(f"完成爬取: {platform_config['name']}")
+
+        self.stdout.write(self.style.SUCCESS("中国政府网所有平台爬取完成"))

core/management/commands/crawl_dongfangyancao.py

@@ -6,15 +6,48 @@ from core.utils import full_site_crawler
 class Command(BaseCommand):
     help = "全站递归爬取 东方烟草报"
 
-    def handle(self, *args, **kwargs):
-        website, created = Website.objects.get_or_create(
-            name="东方烟草报",
-            defaults={
-                'article_list_url': 'https://www.eastobacco.com/',
+    def add_arguments(self, parser):
+        parser.add_argument('--platform', type=str, default='all',
+                            choices=['eastobacco', 'all'],
+                            help='选择爬取平台: eastobacco(东方烟草报), all(全部)')
+
+    def handle(self, *args, **options):
+        platform = options['platform']
+
+        # 东方烟草报各平台配置
+        platforms = {
+            'eastobacco': {
+                'name': '东方烟草报',
+                'base_url': 'https://www.eastobacco.com/',
+                'start_url': 'https://www.eastobacco.com/',
                 'article_selector': 'a'
-            }
-        )
-        start_url = "https://www.eastobacco.com/"
-        self.stdout.write(f"开始全站爬取: {start_url}")
-        full_site_crawler(start_url, website, max_pages=500)
-        self.stdout.write("爬取完成")
+            },
+        }
+
+        if platform == 'all':
+            target_platforms = platforms.values()
+        else:
+            target_platforms = [platforms[platform]]
+
+        for platform_config in target_platforms:
+            website, created = Website.objects.get_or_create(
+                name=platform_config['name'],
+                defaults={
+                    'base_url': platform_config['base_url'],
+                    'article_list_url': platform_config['start_url'],
+                    'article_selector': platform_config['article_selector']
+                }
+            )
+
+            # 确保更新已存在的网站对象的配置
+            if not created:
+                website.base_url = platform_config['base_url']
+                website.article_list_url = platform_config['start_url']
+                website.article_selector = platform_config['article_selector']
+                website.save()
+
+            self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
+            full_site_crawler(platform_config['start_url'], website, max_pages=500)
+            self.stdout.write(f"完成爬取: {platform_config['name']}")
+
+        self.stdout.write(self.style.SUCCESS("东方烟草报所有平台爬取完成"))