nanan/green_classroom
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Support All Platform

Browse Source
This commit is contained in:
yuangyaa
2025-08-14 23:42:16 +08:00
parent 4994310f14
commit ac98ac0057
23 changed files with 267 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/management/commands/crawl_all_media.py
View File

@@ -8,17 +8,17 @@ class Command(BaseCommand):
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--media', type=str, help='指定要爬取的媒体，用逗号分隔') parser.add_argument('--media', type=str, help='指定要爬取的媒体，用逗号分隔')
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
help='指定平台类型: all(全部), web(网站), mobile(移动端)') help='指定平台类型: all(全部), web(网站), mobile(移动端)')
def handle(self, *args, **options): def handle(self, *args, **options):
media_list = options['media'] media_list = options['media']
platform = options['platform'] platform = options['platform']
# 所有中央主流媒体配置 # 所有中央主流媒体配置
all_media = { all_media = {
'rmrb': 'crawl_rmrb', 'rmrb': 'crawl_rmrb',
'xinhua': 'crawl_xinhua', 'xinhua': 'crawl_xinhua',
'cctv': 'crawl_cctv', 'cctv': 'crawl_cctv',
'qiushi': 'crawl_qiushi', 'qiushi': 'crawl_qiushi',
'pla': 'crawl_pla', 'pla': 'crawl_pla',
@@ -39,15 +39,15 @@ class Command(BaseCommand):
'qizhi': 'crawl_qizhi', 'qizhi': 'crawl_qizhi',
'china': 'crawl_china' 'china': 'crawl_china'
} }
# 如果指定了特定媒体，则只爬取指定的媒体 # 如果指定了特定媒体，则只爬取指定的媒体
if media_list: if media_list:
target_media = [media.strip() for media in media_list.split(',')] target_media = [media.strip() for media in media_list.split(',')]
else: else:
target_media = list(all_media.keys()) target_media = list(all_media.keys())
self.stdout.write(f"开始批量爬取 {len(target_media)} 家中央主流媒体...") self.stdout.write(f"开始批量爬取 {len(target_media)} 家中央主流媒体...")
for media in target_media: for media in target_media:
if media in all_media: if media in all_media:
command_name = all_media[media] command_name = all_media[media]
@@ -59,17 +59,17 @@ class Command(BaseCommand):
self.stdout.write(self.style.ERROR(f"爬取 {media} 失败: {e}")) self.stdout.write(self.style.ERROR(f"爬取 {media} 失败: {e}"))
else: else:
self.stdout.write(self.style.WARNING(f"未知媒体: {media}")) self.stdout.write(self.style.WARNING(f"未知媒体: {media}"))
self.stdout.write(self.style.SUCCESS("所有中央主流媒体爬取完成")) self.stdout.write(self.style.SUCCESS("所有中央主流媒体爬取完成"))
# 显示统计信息 # 显示统计信息
total_websites = Website.objects.count() total_websites = Website.objects.count()
total_articles = sum([website.article_set.count() for website in Website.objects.all()]) total_articles = sum([website.article_set.count() for website in Website.objects.all()])
self.stdout.write(f"统计信息:") self.stdout.write(f"统计信息:")
self.stdout.write(f"- 总网站数: {total_websites}") self.stdout.write(f"- 总网站数: {total_websites}")
self.stdout.write(f"- 总文章数: {total_articles}") self.stdout.write(f"- 总文章数: {total_articles}")
# 显示各媒体文章数量 # 显示各媒体文章数量
self.stdout.write(f"各媒体文章数量:") self.stdout.write(f"各媒体文章数量:")
for website in Website.objects.all(): for website in Website.objects.all():

18
core/management/commands/crawl_cctv.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中央广播电视总台及其子网站、客户端、新媒体平台" help = "全站递归爬取 中央广播电视总台及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['cctv', 'cctvnews', 'mobile', 'all'], choices=['cctv', 'cctvnews', 'mobile', 'all'],
help='选择爬取平台: cctv(央视网), cctvnews(央视新闻), mobile(移动端), all(全部)') help='选择爬取平台: cctv(央视网), cctvnews(央视新闻), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中央广播电视总台各平台配置 # 中央广播电视总台各平台配置
platforms = { platforms = {
'cctv': { 'cctv': {
@@ -35,12 +35,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -50,16 +50,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中央广播电视总台所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中央广播电视总台所有平台爬取完成"))

18
core/management/commands/crawl_china.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国网主网及中国网一省份，不转发二级子网站" help = "全站递归爬取 中国网主网及中国网一省份，不转发二级子网站"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['china', 'province', 'all'], choices=['china', 'province', 'all'],
help='选择爬取平台: china(中国网主网), province(中国网一省份), all(全部)') help='选择爬取平台: china(中国网主网), province(中国网一省份), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国网各平台配置 # 中国网各平台配置
platforms = { platforms = {
'china': { 'china': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国网所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国网所有平台爬取完成"))

18
core/management/commands/crawl_chinadaily.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 中国日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['chinadaily', 'mobile', 'all'], choices=['chinadaily', 'mobile', 'all'],
help='选择爬取平台: chinadaily(中国日报), mobile(移动端), all(全部)') help='选择爬取平台: chinadaily(中国日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国日报各平台配置 # 中国日报各平台配置
platforms = { platforms = {
'chinadaily': { 'chinadaily': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国日报所有平台爬取完成"))

18
core/management/commands/crawl_chinanews.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国新闻社及其子网站、客户端、新媒体平台" help = "全站递归爬取 中国新闻社及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['chinanews', 'mobile', 'all'], choices=['chinanews', 'mobile', 'all'],
help='选择爬取平台: chinanews(中国新闻社), mobile(移动端), all(全部)') help='选择爬取平台: chinanews(中国新闻社), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国新闻社各平台配置 # 中国新闻社各平台配置
platforms = { platforms = {
'chinanews': { 'chinanews': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国新闻社所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国新闻社所有平台爬取完成"))

18
core/management/commands/crawl_fzrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 法治日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 法治日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['fzrb', 'mobile', 'all'], choices=['fzrb', 'mobile', 'all'],
help='选择爬取平台: fzrb(法治日报), mobile(移动端), all(全部)') help='选择爬取平台: fzrb(法治日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 法治日报各平台配置 # 法治日报各平台配置
platforms = { platforms = {
'fzrb': { 'fzrb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("法治日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("法治日报所有平台爬取完成"))

18
core/management/commands/crawl_gmrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 光明日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 光明日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['gmrb', 'mobile', 'all'], choices=['gmrb', 'mobile', 'all'],
help='选择爬取平台: gmrb(光明日报), mobile(移动端), all(全部)') help='选择爬取平台: gmrb(光明日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 光明日报各平台配置 # 光明日报各平台配置
platforms = { platforms = {
'gmrb': { 'gmrb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("光明日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("光明日报所有平台爬取完成"))

20
core/management/commands/crawl_grrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 工人日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 工人日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['grrb', 'mobile', 'all'], choices=['grrb', 'mobile', 'all'],
help='选择爬取平台: grrb(工人日报), mobile(移动端), all(全部)') help='选择爬取平台: grrb(工人日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 工人日报各平台配置 # 工人日报各平台配置
platforms = { platforms = {
'grrb': { 'grrb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("工人日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("工人日报所有平台爬取完成"))

18
core/management/commands/crawl_jjrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 经济日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 经济日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['jjrb', 'mobile', 'all'], choices=['jjrb', 'mobile', 'all'],
help='选择爬取平台: jjrb(经济日报), mobile(移动端), all(全部)') help='选择爬取平台: jjrb(经济日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 经济日报各平台配置 # 经济日报各平台配置
platforms = { platforms = {
'jjrb': { 'jjrb': {
@@ -23,12 +23,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
}, },
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -38,16 +38,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("经济日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("经济日报所有平台爬取完成"))

18
core/management/commands/crawl_nmrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 农民日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 农民日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['nmrb', 'mobile', 'all'], choices=['nmrb', 'mobile', 'all'],
help='选择爬取平台: nmrb(农民日报), mobile(移动端), all(全部)') help='选择爬取平台: nmrb(农民日报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 农民日报各平台配置 # 农民日报各平台配置
platforms = { platforms = {
'nmrb': { 'nmrb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("农民日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("农民日报所有平台爬取完成"))

18
core/management/commands/crawl_pla.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 解放军报及其子网站、客户端、新媒体平台" help = "全站递归爬取 解放军报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['pla', 'mobile', 'all'], choices=['pla', 'mobile', 'all'],
help='选择爬取平台: pla(解放军报), mobile(移动端), all(全部)') help='选择爬取平台: pla(解放军报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 解放军报各平台配置 # 解放军报各平台配置
platforms = { platforms = {
'pla': { 'pla': {
@@ -23,12 +23,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
}, },
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -38,16 +38,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("解放军报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("解放军报所有平台爬取完成"))

18
core/management/commands/crawl_qiushi.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 求是杂志及其子网站、客户端、新媒体平台" help = "全站递归爬取 求是杂志及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['qiushi', 'mobile', 'all'], choices=['qiushi', 'mobile', 'all'],
help='选择爬取平台: qiushi(求是网), mobile(移动端), all(全部)') help='选择爬取平台: qiushi(求是网), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 求是杂志各平台配置 # 求是杂志各平台配置
platforms = { platforms = {
'qiushi': { 'qiushi': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("求是杂志所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("求是杂志所有平台爬取完成"))

20
core/management/commands/crawl_qizhi.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 旗帜网及其子网站、客户端、新媒体平台" help = "全站递归爬取 旗帜网及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['qizhi', 'mobile', 'all'], choices=['qizhi', 'mobile', 'all'],
help='选择爬取平台: qizhi(旗帜网), mobile(移动端), all(全部)') help='选择爬取平台: qizhi(旗帜网), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 旗帜网各平台配置 # 旗帜网各平台配置
platforms = { platforms = {
'qizhi': { 'qizhi': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a[href^="/"]' # 修改选择器以更好地匹配文章链接 'article_selector': 'a[href^="/"]' # 修改选择器以更好地匹配文章链接
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("旗帜网所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("旗帜网所有平台爬取完成"))

20
core/management/commands/crawl_rmrb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 人民日报及其子网站、客户端、新媒体平台" help = "全站递归爬取 人民日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['peopleapp', 'people', 'paper', 'all'], choices=['peopleapp', 'people', 'paper', 'all'],
help='选择爬取平台: peopleapp(客户端), people(人民网), paper(报纸), all(全部)') help='选择爬取平台: peopleapp(客户端), people(人民网), paper(报纸), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 人民日报各平台配置 # 人民日报各平台配置
platforms = { platforms = {
'peopleapp': { 'peopleapp': {
@@ -35,12 +35,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -50,16 +50,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("人民日报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("人民日报所有平台爬取完成"))

2
core/management/commands/crawl_rmzxb.py
View File

@@ -50,4 +50,4 @@ class Command(BaseCommand):
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("人民政协网所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("人民政协网所有平台爬取完成"))

18
core/management/commands/crawl_xinhua.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 新华社及其子网站、客户端、新媒体平台" help = "全站递归爬取 新华社及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['news', 'xinhuanet', 'mobile', 'all'], choices=['news', 'xinhuanet', 'mobile', 'all'],
help='选择爬取平台: news(新华网), xinhuanet(新华网主站), mobile(移动端), all(全部)') help='选择爬取平台: news(新华网), xinhuanet(新华网主站), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 新华社各平台配置 # 新华社各平台配置
platforms = { platforms = {
'news': { 'news': {
@@ -35,12 +35,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -50,16 +50,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("新华社所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("新华社所有平台爬取完成"))

18
core/management/commands/crawl_xuexi.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 学习强国中央媒体学习号及省级以上学习平台" help = "全站递归爬取 学习强国中央媒体学习号及省级以上学习平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['xuexi', 'central', 'provincial', 'all'], choices=['xuexi', 'central', 'provincial', 'all'],
help='选择爬取平台: xuexi(学习强国主站), central(中央媒体), provincial(省级平台), all(全部)') help='选择爬取平台: xuexi(学习强国主站), central(中央媒体), provincial(省级平台), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 学习强国各平台配置 # 学习强国各平台配置
platforms = { platforms = {
'xuexi': { 'xuexi': {
@@ -35,12 +35,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -50,16 +50,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("学习强国所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("学习强国所有平台爬取完成"))

18
core/management/commands/crawl_xxsb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 学习时报及其子网站、客户端、新媒体平台" help = "全站递归爬取 学习时报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['xxsb', 'mobile', 'all'], choices=['xxsb', 'mobile', 'all'],
help='选择爬取平台: xxsb(学习时报), mobile(移动端), all(全部)') help='选择爬取平台: xxsb(学习时报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 学习时报各平台配置 # 学习时报各平台配置
platforms = { platforms = {
'xxsb': { 'xxsb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("学习时报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("学习时报所有平台爬取完成"))

18
core/management/commands/crawl_zgfnb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国妇女报及其子网站、客户端、新媒体平台" help = "全站递归爬取 中国妇女报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['zgfnb', 'mobile', 'all'], choices=['zgfnb', 'mobile', 'all'],
help='选择爬取平台: zgfnb(中国妇女报), mobile(移动端), all(全部)') help='选择爬取平台: zgfnb(中国妇女报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国妇女报各平台配置 # 中国妇女报各平台配置
platforms = { platforms = {
'zgfnb': { 'zgfnb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国妇女报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国妇女报所有平台爬取完成"))

18
core/management/commands/crawl_zgjwjc.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国纪检监察报及其子网站、客户端、新媒体平台" help = "全站递归爬取 中国纪检监察报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['zgjwjc', 'mobile', 'all'], choices=['zgjwjc', 'mobile', 'all'],
help='选择爬取平台: zgjwjc(中国纪检监察报), mobile(移动端), all(全部)') help='选择爬取平台: zgjwjc(中国纪检监察报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国纪检监察报各平台配置 # 中国纪检监察报各平台配置
platforms = { platforms = {
'zgjwjc': { 'zgjwjc': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国纪检监察报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国纪检监察报所有平台爬取完成"))

18
core/management/commands/crawl_zgqnb.py
View File

@@ -7,13 +7,13 @@ class Command(BaseCommand):
help = "全站递归爬取 中国青年报及其子网站、客户端、新媒体平台" help = "全站递归爬取 中国青年报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all', parser.add_argument('--platform', type=str, default='all',
choices=['zgqnb', 'mobile', 'all'], choices=['zgqnb', 'mobile', 'all'],
help='选择爬取平台: zgqnb(中国青年报), mobile(移动端), all(全部)') help='选择爬取平台: zgqnb(中国青年报), mobile(移动端), all(全部)')
def handle(self, *args, **options): def handle(self, *args, **options):
platform = options['platform'] platform = options['platform']
# 中国青年报各平台配置 # 中国青年报各平台配置
platforms = { platforms = {
'zgqnb': { 'zgqnb': {
@@ -29,12 +29,12 @@ class Command(BaseCommand):
'article_selector': 'a' 'article_selector': 'a'
} }
} }
if platform == 'all': if platform == 'all':
target_platforms = platforms.values() target_platforms = platforms.values()
else: else:
target_platforms = [platforms[platform]] target_platforms = [platforms[platform]]
for platform_config in target_platforms: for platform_config in target_platforms:
website, created = Website.objects.get_or_create( website, created = Website.objects.get_or_create(
name=platform_config['name'], name=platform_config['name'],
@@ -44,16 +44,16 @@ class Command(BaseCommand):
'article_selector': platform_config['article_selector'] 'article_selector': platform_config['article_selector']
} }
) )
# 确保更新已存在的网站对象的配置 # 确保更新已存在的网站对象的配置
if not created: if not created:
website.base_url = platform_config['base_url'] website.base_url = platform_config['base_url']
website.article_list_url = platform_config['start_url'] website.article_list_url = platform_config['start_url']
website.article_selector = platform_config['article_selector'] website.article_selector = platform_config['article_selector']
website.save() website.save()
self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}") self.stdout.write(f"开始爬取: {platform_config['name']} - {platform_config['start_url']}")
full_site_crawler(platform_config['start_url'], website, max_pages=500) full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}") self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国青年报所有平台爬取完成")) self.stdout.write(self.style.SUCCESS("中国青年报所有平台爬取完成"))

3
core/templates/core/article_detail.html
View File

@@ -77,7 +77,8 @@
<div class="meta"> <div class="meta">
网站: {{ article.website.name }} | 网站: {{ article.website.name }} |
发布时间: {{ article.pub_date|date:"Y-m-d H:i" }} | 发布时间: {{ article.pub_date|date:"Y-m-d H:i" }} |
创建时间: {{ article.created_at|date:"Y-m-d H:i" }} 创建时间: {{ article.created_at|date:"Y-m-d H:i" }} |
源网址: <a href="{{ article.url }}" target="_blank">{{ article.url }}</a>
</div> </div>
<div class="content"> <div class="content">

107
core/utils.py
View File

@@ -126,7 +126,8 @@ def process_article(url, website):
soup.find("div", class_="content") or soup.find("div", class_="content") or
soup.find("div", id="content") or soup.find("div", id="content") or
soup.find("div", class_="article") or soup.find("div", class_="article") or
soup.find("div", class_="main-content") soup.find("div", class_="main-content") or
soup.find("span", id="detailContent") # 添加新华网特有的内容容器
) )
elif website.name == "东方烟草报": elif website.name == "东方烟草报":
# 优化东方烟草报的标题提取逻辑，按优先级尝试多种选择器 # 优化东方烟草报的标题提取逻辑，按优先级尝试多种选择器
@@ -177,45 +178,45 @@ def process_article(url, website):
soup.find("div", class_="rm_txt_con") or # 添加人民网特有的内容容器 soup.find("div", class_="rm_txt_con") or # 添加人民网特有的内容容器
soup.find("div", class_="text_c") # 添加新的内容容器 soup.find("div", class_="text_c") # 添加新的内容容器
) )
# 针对人民网的特殊处理，清理内容中的无关元素 # 针对人民网的特殊处理，清理内容中的无关元素
if content_tag: if content_tag:
# 移除编辑信息 # 移除编辑信息
for editor_element in content_tag.find_all("div", class_="edit"): for editor_element in content_tag.find_all("div", class_="edit"):
editor_element.decompose() editor_element.decompose()
# 移除分享相关元素 # 移除分享相关元素
for share_element in content_tag.find_all("p", class_="paper_num"): for share_element in content_tag.find_all("p", class_="paper_num"):
share_element.decompose() share_element.decompose()
# 移除无关的box_pic元素 # 移除无关的box_pic元素
for pic_element in content_tag.find_all("div", class_="box_pic"): for pic_element in content_tag.find_all("div", class_="box_pic"):
pic_element.decompose() pic_element.decompose()
# 移除无关的zdfy元素 # 移除无关的zdfy元素
for zdfy_element in content_tag.find_all("div", class_="zdfy"): for zdfy_element in content_tag.find_all("div", class_="zdfy"):
zdfy_element.decompose() zdfy_element.decompose()
# 移除无关的center元素 # 移除无关的center元素
for center_element in content_tag.find_all("center"): for center_element in content_tag.find_all("center"):
center_element.decompose() center_element.decompose()
# 移除无关的bza元素 # 移除无关的bza元素
for bza_element in content_tag.find_all("div", class_="bza"): for bza_element in content_tag.find_all("div", class_="bza"):
bza_element.decompose() bza_element.decompose()
# 移除隐藏的无关元素 # 移除隐藏的无关元素
for hidden_element in content_tag.find_all(attrs={"style": "display: none;"}): for hidden_element in content_tag.find_all(attrs={"style": "display: none;"}):
hidden_element.decompose() hidden_element.decompose()
# 移除相关专题 # 移除相关专题
for related_element in content_tag.find_all("div", id="rwb_tjyd"): for related_element in content_tag.find_all("div", id="rwb_tjyd"):
related_element.decompose() related_element.decompose()
# 移除推荐阅读 # 移除推荐阅读
for recommend_element in content_tag.find_all("div", class_="clearfix box_cai"): for recommend_element in content_tag.find_all("div", class_="clearfix box_cai"):
recommend_element.decompose() recommend_element.decompose()
# 移除相关专题列表 # 移除相关专题列表
for topic_element in content_tag.find_all("div", class_="clearfix text_like"): for topic_element in content_tag.find_all("div", class_="clearfix text_like"):
topic_element.decompose() topic_element.decompose()
@@ -414,7 +415,7 @@ def process_article(url, website):
title_text = first_p.find("strong").get_text().strip() title_text = first_p.find("strong").get_text().strip()
# 创建一个虚拟的title_tag对象 # 创建一个虚拟的title_tag对象
title_tag = first_p.find("strong") title_tag = first_p.find("strong")
content_tag = ( content_tag = (
soup.find("div", class_="content") or soup.find("div", class_="content") or
soup.find("div", class_="article-content") or soup.find("div", class_="article-content") or
@@ -425,27 +426,28 @@ def process_article(url, website):
soup.find("div", class_="article-body") or soup.find("div", class_="article-body") or
soup.find("div", class_="text_box") # 添加人民政协网特有内容容器 soup.find("div", class_="text_box") # 添加人民政协网特有内容容器
) )
# 针对人民政协网的特殊处理，清理内容中的无关元素 # 针对人民政协网的特殊处理，清理内容中的无关元素
if content_tag: if content_tag:
# 移除编辑信息 # 移除编辑信息
for editor_element in content_tag.find_all("p", class_="Editor"): for editor_element in content_tag.find_all("p", class_="Editor"):
editor_element.decompose() editor_element.decompose()
# 移除分享相关元素 # 移除分享相关元素
for share_element in content_tag.find_all("div", class_="share"): for share_element in content_tag.find_all("div", class_="share"):
share_element.decompose() share_element.decompose()
# 移除Remark元素 # 移除Remark元素
for remark_element in content_tag.find_all("div", class_="Remark"): for remark_element in content_tag.find_all("div", class_="Remark"):
remark_element.decompose() remark_element.decompose()
# 移除Paging元素 # 移除Paging元素
for paging_element in content_tag.find_all("div", class_="Paging"): for paging_element in content_tag.find_all("div", class_="Paging"):
paging_element.decompose() paging_element.decompose()
# 移除政协号客户端下载提示 # 移除政协号客户端下载提示
for zxh_element in content_tag.find_all("div", style=lambda x: x and "background:#F9F9F9;padding:50px" in x): for zxh_element in content_tag.find_all("div",
style=lambda x: x and "background:#F9F9F9;padding:50px" in x):
zxh_element.decompose() zxh_element.decompose()
# 移除版权信息 # 移除版权信息
@@ -503,6 +505,7 @@ def process_article(url, website):
soup.find("title") soup.find("title")
) )
content_tag = ( content_tag = (
soup.find("div", id="detail") or # 添加学习时报特有内容容器
soup.find("div", class_="content") or soup.find("div", class_="content") or
soup.find("div", class_="article-content") or soup.find("div", class_="article-content") or
soup.find("div", id="content") or soup.find("div", id="content") or
@@ -511,6 +514,24 @@ def process_article(url, website):
soup.find("div", class_="article") or soup.find("div", class_="article") or
soup.find("div", class_="article-body") soup.find("div", class_="article-body")
) )
# 针对学习时报的特殊处理，清理内容中的无关元素
if content_tag:
# 移除编辑信息
for editor_element in content_tag.find_all("div", class_="editor"):
editor_element.decompose()
# 移除分享相关元素
for share_element in content_tag.find_all("div", class_="share"):
share_element.decompose()
# 移除无关的TRS_Editor包装层
for trs_editor in content_tag.find_all("div", class_="TRS_Editor"):
trs_editor.unwrap() # unwrap只移除标签，保留内容
# 移除Custom_UnionStyle包装层
for custom_style in content_tag.find_all("div", class_="Custom_UnionStyle"):
custom_style.unwrap() # unwrap只移除标签，保留内容
elif "中国青年报" in website.name or "cyol" in website.name: elif "中国青年报" in website.name or "cyol" in website.name:
# 中国青年报的文章结构处理 - 修复无法爬取问题 # 中国青年报的文章结构处理 - 修复无法爬取问题
title_tag = ( title_tag = (
@@ -532,6 +553,7 @@ def process_article(url, website):
title_tag = ( title_tag = (
soup.find("h1", class_="title") or soup.find("h1", class_="title") or
soup.find("h1") or soup.find("h1") or
soup.find("p", class_="f_container_title") or # 添加中国妇女报特有标题容器
soup.find("title") soup.find("title")
) )
content_tag = ( content_tag = (
@@ -541,7 +563,9 @@ def process_article(url, website):
soup.find("div", class_="text") or soup.find("div", class_="text") or
soup.find("div", class_="main-content") or soup.find("div", class_="main-content") or
soup.find("div", class_="article") or soup.find("div", class_="article") or
soup.find("div", class_="article-body") soup.find("div", class_="article-body") or
soup.find("div", class_="f_container_left") or # 添加中国妇女报特有内容容器
soup.find("div", class_="f_container") # 添加另一种可能的内容容器
) )
elif "法治日报" in website.name or "legaldaily" in website.name: elif "法治日报" in website.name or "legaldaily" in website.name:
# 法治日报的文章结构处理 - 修复无法爬取问题 # 法治日报的文章结构处理 - 修复无法爬取问题
@@ -604,7 +628,7 @@ def process_article(url, website):
elif "旗帜网" in website.name or "qizhiwang" in website.name: elif "旗帜网" in website.name or "qizhiwang" in website.name:
# 旗帜网的文章结构处理 - 修复不保存文章内容问题 # 旗帜网的文章结构处理 - 修复不保存文章内容问题
title_tag = ( title_tag = (
soup.find("div", class_="w1200 flag-text-tit clearfix") and soup.find("div", class_="w1200 flag-text-tit clearfix") and
soup.find("div", class_="w1200 flag-text-tit clearfix").find("h1") or soup.find("div", class_="w1200 flag-text-tit clearfix").find("h1") or
soup.find("h1", class_="title") or soup.find("h1", class_="title") or
soup.find("h1") or soup.find("h1") or
@@ -620,29 +644,29 @@ def process_article(url, website):
soup.find("div", class_="article") or soup.find("div", class_="article") or
soup.find("div", class_="article-body") soup.find("div", class_="article-body")
) )
# 针对旗帜网的特殊处理，清理内容中的无关元素 # 针对旗帜网的特殊处理，清理内容中的无关元素
if content_tag: if content_tag:
# 移除编辑信息 # 移除编辑信息
for editor_element in content_tag.find_all("p", class_="editor"): for editor_element in content_tag.find_all("p", class_="editor"):
editor_element.decompose() editor_element.decompose()
# 移除分享相关元素 # 移除分享相关元素
for share_element in content_tag.find_all("div", class_="share-demo"): for share_element in content_tag.find_all("div", class_="share-demo"):
share_element.decompose() share_element.decompose()
# 移除文字缩放相关元素 # 移除文字缩放相关元素
for scale_element in content_tag.find_all("div", class_="scale-main"): for scale_element in content_tag.find_all("div", class_="scale-main"):
scale_element.decompose() scale_element.decompose()
# 移除无关的div.pic元素 # 移除无关的div.pic元素
for pic_element in content_tag.find_all("div", class_="pic"): for pic_element in content_tag.find_all("div", class_="pic"):
pic_element.decompose() pic_element.decompose()
# 移除无关的zdfy元素 # 移除无关的zdfy元素
for zdfy_element in content_tag.find_all("div", class_="zdfy"): for zdfy_element in content_tag.find_all("div", class_="zdfy"):
zdfy_element.decompose() zdfy_element.decompose()
# 移除无关的center元素 # 移除无关的center元素
for center_element in content_tag.find_all("center"): for center_element in content_tag.find_all("center"):
center_element.decompose() center_element.decompose()
@@ -714,6 +738,28 @@ def process_article(url, website):
if not src: if not src:
src = video.get("data-url") or video.get("data-video") src = video.get("data-url") or video.get("data-video")
# 新增：检查新华网特有的视频源属性
if not src:
src = video.get("data-video-src")
# 新增：针对新华网的特殊处理，从复杂播放器结构中提取视频源
if not src and "新华网" in website.name:
# 尝试从video标签的属性中直接获取src
for attr in video.attrs:
if 'src' in attr.lower():
src = video.attrs.get(attr)
break
# 如果还是没有找到，尝试查找父容器中的视频源信息
if not src:
parent = video.parent
if parent and parent.name == 'div' and 'player-container' in parent.get('class', []):
# 检查是否有data-*属性包含视频信息
for attr, value in parent.attrs.items():
if 'data' in attr and isinstance(value, str) and ('.mp4' in value or 'video' in value):
src = value
break
if not src: if not src:
continue continue
@@ -726,6 +772,10 @@ def process_article(url, website):
if "cctv.com" in src or "cntv.cn" in src: if "cctv.com" in src or "cntv.cn" in src:
print(f"发现央视视频: {src}") print(f"发现央视视频: {src}")
# 针对新华网的特殊处理
elif "新华网" in website.name:
print(f"发现新华网视频: {src}")
local_path = download_media(src, save_dir) local_path = download_media(src, save_dir)
if local_path: if local_path:
rel_path = os.path.relpath(local_path, settings.MEDIA_ROOT) rel_path = os.path.relpath(local_path, settings.MEDIA_ROOT)
@@ -812,6 +862,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
soup.find("div", id="content") is not None or soup.find("div", id="content") is not None or
soup.find("div", class_="article") is not None or soup.find("div", class_="article") is not None or
soup.find("div", class_="main-content") is not None or soup.find("div", class_="main-content") is not None or
soup.find("span", id="detailContent") is not None or # 添加新华网特有内容容器判断
("/news/" in path) or ("/news/" in path) or
("/article/" in path) or ("/article/" in path) or
(path.startswith("/detail/") and len(path) > 10) (path.startswith("/detail/") and len(path) > 10)
@@ -1064,6 +1115,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
soup.find("div", class_="main-content") is not None or soup.find("div", class_="main-content") is not None or
soup.find("div", class_="article") is not None or soup.find("div", class_="article") is not None or
soup.find("div", class_="article-body") is not None or soup.find("div", class_="article-body") is not None or
soup.find("div", id="detail") is not None or # 添加学习时报特有内容容器判断
("/article/" in path) or ("/article/" in path) or
("/content/" in path) or ("/content/" in path) or
(path.startswith("/detail/") and len(path) > 10) (path.startswith("/detail/") and len(path) > 10)
@@ -1168,7 +1220,6 @@ def full_site_crawler(start_url, website, max_pages=1000):
soup.find("div", class_="article-content") is not None or soup.find("div", class_="article-content") is not None or
(soup.find("div", id="content") is not None and (soup.find("div", id="content") is not None and
soup.find("h1") is not None) or soup.find("h1") is not None) or
soup.find("div", class_="text") is not None or
soup.find("div", class_="main-content") is not None or soup.find("div", class_="main-content") is not None or
soup.find("div", class_="article") is not None or soup.find("div", class_="article") is not None or
soup.find("div", class_="article-body") is not None or soup.find("div", class_="article-body") is not None or
@@ -1220,7 +1271,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
if ("/article/" in href_path or if ("/article/" in href_path or
href_path.startswith("/detail/") or href_path.startswith("/detail/") or
("/dynamic/" in href_path and "article" in href_path) or ("/dynamic/" in href_path and "article" in href_path) or
href_path.count("/") > 2): # 更深层的页面可能是文章页 href_path.count("/") > 2): # 更深层
queue.append(href) queue.append(href)
elif href not in visited and is_valid_url(href, base_netloc): elif href not in visited and is_valid_url(href, base_netloc):
queue.append(href) queue.append(href)