from django.core.management.base import BaseCommand
from core.models import Website
from core.utils import full_site_crawler


class Command(BaseCommand):
    help = "全站递归爬取 东方烟草报"

    def handle(self, *args, **kwargs):
        website, created = Website.objects.get_or_create(
            name="东方烟草报",
            defaults={
                'article_list_url': 'https://www.eastobacco.com/',
                'article_selector': 'a'
            }
        )
        start_url = "https://www.eastobacco.com/"
        self.stdout.write(f"开始全站爬取: {start_url}")
        full_site_crawler(start_url, website, max_pages=500)
        self.stdout.write("爬取完成")