nanan/green_classroom
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix bugs and support all platform

Browse Source
This commit is contained in:
yuangyaa
2025-08-15 08:33:47 +08:00
parent e82b85f4dd
commit 4945b4c6b0
36 changed files with 2296 additions and 992 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/management/commands/crawl_all_media.py
View File

@@ -9,7 +9,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--media', type=str, help='指定要爬取的媒体，用逗号分隔')
parser.add_argument('--platform', type=str, default='all',
help='指定平台类型: all(全部), web(网站), mobile(移动端)')
help='指定平台类型: all(全部), web(网站)')
def handle(self, *args, **options):
media_list = options['media']

2
core/management/commands/crawl_cctv.py
View File

@@ -9,7 +9,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['cctv', 'cctvnews', 'mobile', 'all'],
choices=['cctv', 'cctvnews', 'all'],
help='选择爬取平台: cctv(央视网), cctvnews(央视新闻), all(全部)')
def handle(self, *args, **options):

10
core/management/commands/crawl_china.py
View File

@@ -3,13 +3,12 @@ from core.models import Website
from core.utils import full_site_crawler
# jimmy.fang-20250815: 因URL问题，移除中国网-省份
class Command(BaseCommand):
help = "全站递归爬取 中国网主网及中国网一省份，不转发二级子网站"
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['china', 'province', 'all'],
choices=['china', 'all'],
help='选择爬取平台: china(中国网主网), province(中国网一省份), all(全部)')
def handle(self, *args, **options):
@@ -23,12 +22,7 @@ class Command(BaseCommand):
'start_url': 'http://www.china.com.cn',
'article_selector': 'a'
},
# 'province': {
# 'name': '中国网一省份',
# 'base_url': 'http://www.china.com.cn',
# 'start_url': 'http://www.china.com.cn/province',
# 'article_selector': 'a'
# }
}
if platform == 'all':

2
core/management/commands/crawl_chinanews.py
View File

@@ -8,7 +8,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['chinanews', 'mobile', 'all'],
choices=['chinanews', 'all'],
help='选择爬取平台: chinanews(中国新闻社), all(全部)')
def handle(self, *args, **options):

2
core/management/commands/crawl_cngov.py
View File

@@ -50,4 +50,4 @@ class Command(BaseCommand):
full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("中国政府网所有平台爬取完成"))
self.stdout.write(self.style.SUCCESS("中国政府网所有平台爬取完成"))

2
core/management/commands/crawl_dongfangyancao.py
View File

@@ -50,4 +50,4 @@ class Command(BaseCommand):
full_site_crawler(platform_config['start_url'], website, max_pages=500)
self.stdout.write(f"完成爬取: {platform_config['name']}")
self.stdout.write(self.style.SUCCESS("东方烟草报所有平台爬取完成"))
self.stdout.write(self.style.SUCCESS("东方烟草报所有平台爬取完成"))

2
core/management/commands/crawl_fzrb.py
View File

@@ -8,7 +8,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['fzrb', 'mobile', 'all'],
choices=['fzrb', 'all'],
help='选择爬取平台: fzrb(法治日报), all(全部)')
def handle(self, *args, **options):

5
core/management/commands/crawl_gmrb.py
View File

@@ -2,13 +2,14 @@ from django.core.management.base import BaseCommand
from core.models import Website
from core.utils import full_site_crawler
# jimmy.fang-20250815: 光明日报反爬，会被阻挡
# jimmy.fang-20250815: 取消对光明日报的支持，光明日报反爬，被阻挡
class Command(BaseCommand):
help = "全站递归爬取 光明日报及其子网站、客户端、新媒体平台"
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['gmrb', 'mobile', 'all'],
choices=['gmrb', 'all'],
help='选择爬取平台: gmrb(光明日报), all(全部)')
def handle(self, *args, **options):

2
core/management/commands/crawl_grrb.py
View File

@@ -8,7 +8,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['grrb', 'mobile', 'all'],
choices=['grrb', 'all'],
help='选择爬取平台: grrb(工人日报), all(全部)')
def handle(self, *args, **options):

2
core/management/commands/crawl_jjrb.py
View File

@@ -8,7 +8,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['jjrb', 'mobile', 'all'],
choices=['jjrb', 'all'],
help='选择爬取平台: jjrb(经济日报), all(全部)')
def handle(self, *args, **options):

2
core/management/commands/crawl_kjrb.py
View File

@@ -9,7 +9,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['kjrb', 'mobile', 'all'],
choices=['kjrb', 'all'],
help='选择爬取平台: kjrb(科技日报), all(全部)')
def handle(self, *args, **options):

2
core/management/commands/crawl_nmrb.py
View File

@@ -8,7 +8,7 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['nmrb', 'mobile', 'all'],
choices=['nmrb', 'all'],
help='选择爬取平台: nmrb(农民日报), all(全部)')
def handle(self, *args, **options):

4
core/management/commands/crawl_pla.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['pla', 'mobile', 'all'],
help='选择爬取平台: pla(解放军报), mobile(移动端), all(全部)')
choices=['pla', 'all'],
help='选择爬取平台: pla(解放军报), all(全部)')
def handle(self, *args, **options):
platform = options['platform']

4
core/management/commands/crawl_rmzxb.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['rmzxb', 'mobile', 'all'],
help='选择爬取平台: rmzxb(人民政协网), mobile(移动端), all(全部)')
choices=['rmzxb', 'all'],
help='选择爬取平台: rmzxb(人民政协网), all(全部)')
def handle(self, *args, **options):
platform = options['platform']

17
core/management/commands/crawl_xinhua.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['news', 'xinhuanet', 'mobile', 'all'],
help='选择爬取平台: news(新华网), xinhuanet(新华网主站), mobile(移动端), all(全部)')
choices=['news', 'all'],
help='选择爬取平台: news(新华网), all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -22,18 +22,7 @@ class Command(BaseCommand):
'start_url': 'https://www.news.cn',
'article_selector': 'a'
},
'xinhuanet': {
'name': '新华网主站',
'base_url': 'https://www.xinhuanet.com',
'start_url': 'https://www.xinhuanet.com',
'article_selector': 'a'
},
'mobile': {
'name': '新华社移动端',
'base_url': 'https://m.xinhuanet.com',
'start_url': 'https://m.xinhuanet.com',
'article_selector': 'a'
}
}
if platform == 'all':

16
core/management/commands/crawl_xuexi.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['xuexi', 'central', 'provincial', 'all'],
help='选择爬取平台: xuexi(学习强国主站), central(中央媒体), provincial(省级平台), all(全部)')
choices=['xuexi', 'all'],
help='选择爬取平台: xuexi(学习强国主站), all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -22,18 +22,6 @@ class Command(BaseCommand):
'start_url': 'https://www.xuexi.cn',
'article_selector': 'a'
},
'central': {
'name': '学习强国中央媒体',
'base_url': 'https://www.xuexi.cn',
'start_url': 'https://www.xuexi.cn/central',
'article_selector': 'a'
},
'provincial': {
'name': '学习强国省级平台',
'base_url': 'https://www.xuexi.cn',
'start_url': 'https://www.xuexi.cn/provincial',
'article_selector': 'a'
}
}
if platform == 'all':

10
core/management/commands/crawl_xxsb.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['xxsb', 'mobile', 'all'],
help='选择爬取平台: xxsb(学习时报), mobile(移动端), all(全部)')
choices=['xxsb', 'all'],
help='选择爬取平台: xxsb(学习时报),all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -22,12 +22,6 @@ class Command(BaseCommand):
'start_url': 'http://www.studytimes.cn',
'article_selector': 'a'
},
'mobile': {
'name': '学习时报移动端',
'base_url': 'http://m.studytimes.cn',
'start_url': 'http://m.studytimes.cn',
'article_selector': 'a'
}
}
if platform == 'all':

11
core/management/commands/crawl_zgfnb.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['zgfnb', 'mobile', 'all'],
help='选择爬取平台: zgfnb(中国妇女报), mobile(移动端), all(全部)')
choices=['zgfnb', 'all'],
help='选择爬取平台: zgfnb(中国妇女报), all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -22,12 +22,7 @@ class Command(BaseCommand):
'start_url': 'http://www.cnwomen.com.cn',
'article_selector': 'a'
},
'mobile': {
'name': '中国妇女报移动端',
'base_url': 'http://m.cnwomen.com.cn',
'start_url': 'http://m.cnwomen.com.cn',
'article_selector': 'a'
}
}
if platform == 'all':

14
core/management/commands/crawl_zgjwjc.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['zgjwjc', 'mobile', 'all'],
help='选择爬取平台: zgjwjc(中国纪检监察报), mobile(移动端), all(全部)')
choices=['zgjwjc', 'all'],
help='选择爬取平台: zgjwjc(中国纪检监察报),all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -18,16 +18,10 @@ class Command(BaseCommand):
platforms = {
'zgjwjc': {
'name': '中国纪检监察报',
'base_url': 'http://www.jjjcb.cn',
'start_url': 'http://www.jjjcb.cn',
'base_url': 'https://jjjcb.ccdi.gov.cn',
'start_url': 'https://jjjcb.ccdi.gov.cn',
'article_selector': 'a'
},
'mobile': {
'name': '中国纪检监察报移动端',
'base_url': 'http://m.jjjcb.cn',
'start_url': 'http://m.jjjcb.cn',
'article_selector': 'a'
}
}
if platform == 'all':

11
core/management/commands/crawl_zgqnb.py
View File

@@ -8,8 +8,8 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--platform', type=str, default='all',
choices=['zgqnb', 'mobile', 'all'],
help='选择爬取平台: zgqnb(中国青年报), mobile(移动端), all(全部)')
choices=['zgqnb', 'all'],
help='选择爬取平台: zgqnb(中国青年报), all(全部)')
def handle(self, *args, **options):
platform = options['platform']
@@ -22,12 +22,7 @@ class Command(BaseCommand):
'start_url': 'https://www.cyol.com',
'article_selector': 'a'
},
'mobile': {
'name': '中国青年报移动端',
'base_url': 'https://m.cyol.com',
'start_url': 'https://m.cyol.com',
'article_selector': 'a'
}
}
if platform == 'all':

277
core/management/commands/export_articles.py
View File

@@ -6,6 +6,10 @@ import os
from django.conf import settings
import zipfile
from django.utils import timezone
from bs4 import BeautifulSoup
# 添加python-docx库支持
import io
from docx import Document
class Command(BaseCommand):
@@ -119,201 +123,100 @@ class Command(BaseCommand):
# 添加Word格式导出方法
def export_as_word(self, articles_data, output_path):
try:
from docx import Document
from docx.shared import Inches
except ImportError:
self.stdout.write(self.style.ERROR('缺少python-docx库，请安装: pip install python-docx'))
return
# 创建Word文档
doc = Document()
doc.add_heading('文章导出', 0)
for article_data in articles_data:
# 添加文章标题
doc.add_heading(article_data['title'], level=1)
# 添加文章元数据
doc.add_paragraph(f"网站: {article_data['website']}")
doc.add_paragraph(f"URL: {article_data['url']}")
doc.add_paragraph(f"发布时间: {article_data['pub_date']}")
doc.add_paragraph(f"创建时间: {article_data['created_at']}")
# 添加文章内容
doc.add_heading('内容', level=2)
# 简单处理HTML内容，移除标签
from bs4 import BeautifulSoup
soup = BeautifulSoup(article_data['content'], 'html.parser')
# 处理内容中的图片
for img in soup.find_all('img'):
src = img.get('src', '')
if src:
# 尝试添加图片到文档
try:
import os
from django.conf import settings
import requests
from io import BytesIO
# 构建完整的图片路径
if src.startswith('http'):
# 网络图片
response = requests.get(src, timeout=10)
image_stream = BytesIO(response.content)
doc.add_picture(image_stream, width=Inches(4.0))
else:
# 本地图片
full_path = os.path.join(settings.MEDIA_ROOT, src.lstrip('/'))
if os.path.exists(full_path):
doc.add_picture(full_path, width=Inches(4.0))
except Exception as e:
# 如果添加图片失败，添加图片URL作为文本
doc.add_paragraph(f"[图片: {src}]")
# 移除原始img标签
img.decompose()
content_text = soup.get_text()
doc.add_paragraph(content_text)
# 添加媒体文件信息
if article_data['media_files']:
doc.add_heading('媒体文件', level=2)
for media_file in article_data['media_files']:
try:
import os
from django.conf import settings
from io import BytesIO
import requests
full_path = os.path.join(settings.MEDIA_ROOT, media_file)
if os.path.exists(full_path):
# 添加图片到文档
doc.add_picture(full_path, width=Inches(4.0))
else:
# 如果是URL格式的媒体文件
if media_file.startswith('http'):
response = requests.get(media_file, timeout=10)
image_stream = BytesIO(response.content)
doc.add_picture(image_stream, width=Inches(4.0))
else:
doc.add_paragraph(media_file)
except Exception as e:
doc.add_paragraph(media_file)
# 添加分页符
doc.add_page_break()
# 保存文档
doc.save(output_path)
# 创建一个新的Word文档
document = Document()
document.add_heading('文章导出', 0)
for article_data in articles_data:
# 添加文章标题
document.add_heading(article_data['title'], level=1)
# 添加文章信息
document.add_paragraph(f"网站: {article_data['website']}")
document.add_paragraph(f"URL: {article_data['url']}")
document.add_paragraph(f"发布时间: {article_data['pub_date']}")
document.add_paragraph(f"创建时间: {article_data['created_at']}")
# 添加内容标题
document.add_heading('内容:', level=2)
# 处理HTML内容，移除标签
soup = BeautifulSoup(article_data['content'], 'html.parser')
content_text = soup.get_text()
document.add_paragraph(content_text)
# 添加分页符分隔文章
document.add_page_break()
# 保存文档
document.save(output_path)
self.stdout.write(self.style.SUCCESS(f'成功导出为Word格式: {output_path}'))
except Exception as e:
self.stdout.write(self.style.ERROR(f'导出Word格式失败: {e}'))
def export_with_media(self, articles_data, media_files, output_path, format_type):
# 创建ZIP文件
with zipfile.ZipFile(output_path, 'w') as zipf:
# 添加文章数据文件
data_filename = f'articles.{format_type}'
if format_type == 'json':
json_data = json.dumps(articles_data, ensure_ascii=False, indent=2)
zipf.writestr(data_filename, json_data)
elif format_type == 'csv':
# 创建CSV内容
if articles_data:
import io
csv_buffer = io.StringIO()
fieldnames = ['id', 'title', 'website', 'url', 'pub_date', 'content', 'created_at', 'media_files']
writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames)
writer.writeheader()
for article_data in articles_data:
article_data['media_files'] = ';'.join(article_data['media_files']) if article_data[
'media_files'] else ''
writer.writerow(article_data)
zipf.writestr(data_filename, csv_buffer.getvalue())
elif format_type == 'docx':
# 创建Word文档并保存到ZIP
try:
from docx import Document
from docx.shared import Inches
from io import BytesIO
doc = Document()
doc.add_heading('文章导出', 0)
for article_data in articles_data:
doc.add_heading(article_data['title'], level=1)
# 为每篇文章创建独立的文件夹
for article_data in articles_data:
article_folder = f"article_{article_data['id']}_{article_data['title']}"
# 限制文件夹名称长度并移除非法字符
article_folder = article_folder[:50].rstrip()
article_folder = "".join(c for c in article_folder if c.isalnum() or c in (' ','_','-')).rstrip()
# 添加文章数据文件
if format_type == 'docx':
# 创建Word文档并保存到ZIP
data_filename = f'{article_folder}/article.docx'
try:
# 创建文章信息Word文档
doc = Document()
doc.add_heading(article_data['title'], 0)
# 添加文章信息
doc.add_paragraph(f"网站: {article_data['website']}")
doc.add_paragraph(f"URL: {article_data['url']}")
doc.add_paragraph(f"发布时间: {article_data['pub_date']}")
doc.add_paragraph(f"创建时间: {article_data['created_at']}")
doc.add_heading('内容', level=2)
from bs4 import BeautifulSoup
# 添加内容标题
doc.add_heading('内容:', level=1)
# 处理HTML内容
soup = BeautifulSoup(article_data['content'], 'html.parser')
# 处理内容中的图片
for img in soup.find_all('img'):
src = img.get('src', '')
if src:
# 尝试添加图片到文档
try:
import os
from django.conf import settings
import requests
# 构建完整的图片路径
if src.startswith('http'):
# 网络图片
response = requests.get(src, timeout=10)
image_stream = BytesIO(response.content)
doc.add_picture(image_stream, width=Inches(4.0))
else:
# 本地图片
full_path = os.path.join(settings.MEDIA_ROOT, src.lstrip('/'))
if os.path.exists(full_path):
doc.add_picture(full_path, width=Inches(4.0))
except Exception as e:
# 如果添加图片失败，添加图片URL作为文本
doc.add_paragraph(f"[图片: {src}]")
# 移除原始img标签
img.decompose()
content_text = soup.get_text()
doc.add_paragraph(content_text)
if article_data['media_files']:
doc.add_heading('媒体文件', level=2)
for media_file in article_data['media_files']:
try:
import os
from django.conf import settings
full_path = os.path.join(settings.MEDIA_ROOT, media_file)
if os.path.exists(full_path):
# 添加图片到文档
doc.add_picture(full_path, width=Inches(4.0))
else:
# 如果是URL格式的媒体文件
if media_file.startswith('http'):
response = requests.get(media_file, timeout=10)
image_stream = BytesIO(response.content)
doc.add_picture(image_stream, width=Inches(4.0))
else:
doc.add_paragraph(media_file)
except Exception as e:
doc.add_paragraph(media_file)
doc.add_page_break()
# 将文档保存到内存中再写入ZIP
doc_buffer = BytesIO()
doc.save(doc_buffer)
doc_buffer.seek(0)
zipf.writestr(data_filename, doc_buffer.read())
except ImportError:
zipf.writestr(data_filename, "错误：缺少python-docx库，无法生成Word文档")
# 添加媒体文件
for media_path in media_files:
arcname = os.path.join('media', os.path.relpath(media_path, settings.MEDIA_ROOT))
zipf.write(media_path, arcname)
# 将文档保存到内存中
doc_buffer = io.BytesIO()
doc.save(doc_buffer)
doc_buffer.seek(0)
# 将文档添加到ZIP文件
zipf.writestr(data_filename, doc_buffer.getvalue())
except Exception as e:
error_msg = f"错误：无法生成文章Word文档 - {str(e)}"
zipf.writestr(data_filename, error_msg)
# 添加媒体文件到文章的media子文件夹
if article_data['media_files']:
for media_file in article_data['media_files']:
try:
full_path = os.path.join(settings.MEDIA_ROOT, media_file)
if os.path.exists(full_path):
# 添加媒体文件到ZIP中的media子文件夹
media_filename = f"{article_folder}/media/{os.path.basename(media_file)}"
zipf.write(full_path, media_filename)
else:
# 如果是URL格式的媒体文件
if media_file.startswith('http'):
import requests
from io import BytesIO
response = requests.get(media_file, timeout=10)
image_stream = BytesIO(response.content)
media_filename = f"{article_folder}/media/{os.path.basename(media_file)}"
zipf.writestr(media_filename, image_stream.getvalue())
except Exception as e:
# 错误处理，跳过无法添加的文件
pass