19 lines
732 B
Python
19 lines
732 B
Python
from django.core.management.base import BaseCommand
|
|
from core.models import Website
|
|
from core.utils import crawl_xinhua_list
|
|
|
|
class Command(BaseCommand):
|
|
help = '批量爬取新华网文章'
|
|
|
|
def handle(self, *args, **options):
|
|
list_url = "https://www.news.cn/legal/index.html"
|
|
try:
|
|
website = Website.objects.get(base_url="https://www.news.cn/")
|
|
except Website.DoesNotExist:
|
|
self.stdout.write(self.style.ERROR("网站 https://www.news.cn/ 不存在，请先后台添加"))
|
|
return
|
|
|
|
self.stdout.write(f"开始爬取文章列表页: {list_url}")
|
|
crawl_xinhua_list(list_url, website)
|
|
self.stdout.write(self.style.SUCCESS("批量爬取完成"))
|