Add Support full site

core/management/commands/crawl_full_site.py

@@ -0,0 +1,20 @@
+# core/management/commands/crawl_full_site.py
+from django.core.management.base import BaseCommand
+from core.models import Website
+from core.utils import full_site_crawler
+
+class Command(BaseCommand):
+    help = "全站递归爬取 www.news.cn"
+
+    def handle(self, *args, **kwargs):
+        website, created = Website.objects.get_or_create(
+            name="www.news.cn",
+            defaults={
+                'article_list_url': 'https://www.news.cn/',
+                'article_selector': 'a'
+            }
+        )
+        start_url = "https://www.news.cn/"
+        self.stdout.write(f"开始全站爬取: {start_url}")
+        full_site_crawler(start_url, website, max_pages=500)
+        self.stdout.write("爬取完成")