from django.core.management.base import BaseCommand
from core.models import Website
from core.utils import full_site_crawler


class Command(BaseCommand):
    help = "全站递归爬取 人民日报 https://www.peopleapp.com"

    def handle(self, *args, **kwargs):
        website, created = Website.objects.get_or_create(
            name="人民日报",
            defaults={
                'article_list_url': 'https://www.peopleapp.com/home',
                'article_selector': 'a',
                'base_url': 'https://www.peopleapp.com'
            }
        )
        # 确保更新已存在的网站对象的base_url
        if not created and not website.base_url:
            website.base_url = 'https://www.peopleapp.com'
            website.save()
            
        start_url = "https://www.peopleapp.com/home"
        self.stdout.write(f"开始全站爬取: {start_url}")
        full_site_crawler(start_url, website, max_pages=500)
        self.stdout.write("爬取完成")