# core/management/commands/crawl_xinhua.py
from django.core.management.base import BaseCommand
from core.models import Website
from core.utils import full_site_crawler


class Command(BaseCommand):
    help = "全站递归爬取 www.news.cn"

    def handle(self, *args, **kwargs):
        website, created = Website.objects.get_or_create(
            name="www.news.cn",
            defaults={
                'article_list_url': 'https://www.news.cn/',
                'article_selector': 'a'
            }
        )
        start_url = "https://www.news.cn/"
        self.stdout.write(f"开始全站爬取: {start_url}")
        full_site_crawler(start_url, website, max_pages=500)
        self.stdout.write("爬取完成")