fix bugs

core/management/commands/crawl_cngov.py

@@ -8,7 +8,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         website, created = Website.objects.get_or_create(
-            name="www.gov.cn",
+            name="中国政府网",
             defaults={
                 'article_list_url': 'https://www.gov.cn/',
                 'article_selector': 'a'

core/management/commands/crawl_rmrb.py

@@ -0,0 +1,26 @@
+from django.core.management.base import BaseCommand
+from core.models import Website
+from core.utils import full_site_crawler
+
+
+class Command(BaseCommand):
+    help = "全站递归爬取 人民日报 https://www.peopleapp.com"
+
+    def handle(self, *args, **kwargs):
+        website, created = Website.objects.get_or_create(
+            name="人民日报",
+            defaults={
+                'article_list_url': 'https://www.peopleapp.com/home',
+                'article_selector': 'a',
+                'base_url': 'https://www.peopleapp.com'
+            }
+        )
+        # 确保更新已存在的网站对象的base_url
+        if not created and not website.base_url:
+            website.base_url = 'https://www.peopleapp.com'
+            website.save()
+            
+        start_url = "https://www.peopleapp.com/home"
+        self.stdout.write(f"开始全站爬取: {start_url}")
+        full_site_crawler(start_url, website, max_pages=500)
+        self.stdout.write("爬取完成")

core/management/commands/crawl_xinhua.py

@@ -8,7 +8,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         website, created = Website.objects.get_or_create(
-            name="www.news.cn",
+            name="新华网",
             defaults={
                 'article_list_url': 'https://www.news.cn/',
                 'article_selector': 'a'

core/management/commands/export_articles.py

@@ -4,7 +4,6 @@ import json
 import csv
 import os
 from django.conf import settings
-from django.core.files.storage import default_storage
 import zipfile
 from django.utils import timezone