20 lines
679 B
Python
20 lines
679 B
Python
from django.core.management.base import BaseCommand
|
|
from core.models import Website
|
|
from core.utils import full_site_crawler
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "全站递归爬取 www.gov.cn"
|
|
|
|
def handle(self, *args, **kwargs):
|
|
website, created = Website.objects.get_or_create(
|
|
name="www.gov.cn",
|
|
defaults={
|
|
'article_list_url': 'https://www.gov.cn/',
|
|
'article_selector': 'a'
|
|
}
|
|
)
|
|
start_url = "https://www.gov.cn/"
|
|
self.stdout.write(f"开始全站爬取: {start_url}")
|
|
full_site_crawler(start_url, website, max_pages=500)
|
|
self.stdout.write("爬取完成") |