fix bugs and support all platform

This commit is contained in:
2025-08-15 08:33:47 +08:00
parent e82b85f4dd
commit 4945b4c6b0
36 changed files with 2296 additions and 992 deletions

View File

@@ -2,13 +2,14 @@ from django.core.management.base import BaseCommand
from core.models import Website
from core.utils import full_site_crawler
# jimmy.fang-20250815: 光明日报反爬,会被阻挡
# jimmy.fang-20250815: 取消对光明日报的支持,光明日报反爬,被阻挡
class Command(BaseCommand):
help = "全站递归爬取 光明日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['gmrb', 'mobile', 'all'],
choices=['gmrb', 'all'],
help='选择爬取平台: gmrb(光明日报), all(全部)')
def handle(self, *args, **options):