Add Support the other website

core/management/commands/crawl_xuexi.py

@@ -0,0 +1,65 @@
+from django.core.management.base import BaseCommand
+from core.models import Website
+from core.utils import full_site_crawler
+
+
+class Command(BaseCommand):
+    help = "全站递归爬取 学习强国中央媒体学习号及省级以上学习平台"
+
+    def add_arguments(self, parser):
+        parser.add_argument('--platform', type=str, default='all', 
+                          choices=['xuexi', 'central', 'provincial', 'all'],
+                          help='选择爬取平台: xuexi(学习强国主站), central(中央媒体), provincial(省级平台), all(全部)')
+
+    def handle(self, *args, **options):
+        platform = options['platform']
+        
+        # 学习强国各平台配置
+        platforms = {
+            'xuexi': {
+                'name': '学习强国',
+                'base_url': 'https://www.xuexi.cn',
+                'start_url': 'https://www.xuexi.cn',
+                'article_selector': 'a'
+            },
+            'central': {
+                'name': '学习强国中央媒体',
+                'base_url': 'https://www.xuexi.cn',
+                'start_url': 'https://www.xuexi.cn/central',
+                'article_selector': 'a'
+            },
+            'provincial': {
+                'name': '学习强国省级平台',
+                'base_url': 'https://www.xuexi.cn',
+                'start_url': 'https://www.xuexi.cn/provincial',
+                'article_selector': 'a'
+            }
+        }
+        
+        if platform == 'all':
+            target_platforms = platforms.values()
+        else:
+            target_platforms = [platforms[platform]]
+        
+        for platform_config in target_platforms:
+            website, created = Website.objects.get_or_create(
+                name=platform_config['name'],
+                defaults={
+                    'base_url': platform_config['base_url'],
+                    'article_list_url': platform_config['start_url'],
+                    'article_selector': platform_config['article_selector']
+                }
+            )
+            
+            # 确保更新已存在的网站对象的配置
+            if not created:
+                website.base_url = platform_config['base_url']
+                website.article_list_url = platform_config['start_url']
+                website.article_selector = platform_config['article_selector']
+                website.save()
+            
+            self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
+            full_site_crawler(platform_config['start_url'], website, max_pages=500)
+            self.stdout.write(f"完成爬取: {platform_config['name']}")
+        
+        self.stdout.write(self.style.SUCCESS("学习强国所有平台爬取完成"))