from django.shortcuts import render, get_object_or_404 from django.http import HttpResponse, Http404 from django.db.models import Q, Count from django.conf import settings from django.utils import timezone from django.core.files.storage import default_storage from .models import Website, CrawlTask, CrawledContent, CrawlLog, SearchKeyword, MediaFile from rest_framework import viewsets, filters from rest_framework.decorators import action from rest_framework.response import Response from .serializers import ( WebsiteSerializer, CrawlTaskSerializer, CrawledContentSerializer, CrawlLogSerializer, SearchKeywordSerializer ) import json from django.core.paginator import Paginator from django.db.models.functions import TruncDate from django.db.models import Count import os import tempfile import zipfile from io import BytesIO from docx import Document from django.core.files.base import ContentFile def dashboard(request): """仪表板视图""" # 统计数据 total_websites = Website.objects.filter(is_active=True).count() total_tasks = CrawlTask.objects.count() total_contents = CrawledContent.objects.count() active_tasks = CrawlTask.objects.filter(status='running').count() # 获取所有网站 websites = Website.objects.filter(is_active=True).order_by('name') # 获取当前选中的网站ID selected_website_id = request.GET.get('website') # 获取分页参数 page_number = request.GET.get('page', 1) page_size = request.GET.get('page_size', 20) # 默认每页20篇文章 # 尝试转换page_size为整数 try: page_size = int(page_size) # 限制page_size在合理范围内 page_size = max(10, min(100, page_size)) except (ValueError, TypeError): page_size = 20 # 获取所有爬取的内容，按网站和创建时间排序 all_contents = CrawledContent.objects.select_related('website').order_by('website__name', '-created_at') # 如果选择了特定网站，则进行过滤 if selected_website_id: try: selected_website_id = int(selected_website_id) all_contents = all_contents.filter(website_id=selected_website_id) except (ValueError, TypeError): pass # 分页处理 paginator = Paginator(all_contents, page_size) page_obj = paginator.get_page(page_number) # 按网站分组内容 contents_by_website = {} for content in page_obj: website_name = content.website.name if website_name not in contents_by_website: contents_by_website[website_name] = [] contents_by_website[website_name].append(content) # 最近的任务 recent_tasks = CrawlTask.objects.order_by('-created_at')[:5] # 媒体文件统计 total_media_files = CrawledContent.objects.aggregate( total_media=Count('media_files') )['total_media'] or 0 stats = { 'total_websites': total_websites, 'total_tasks': total_tasks, 'total_contents': total_contents, 'active_tasks': active_tasks, 'websites': websites, 'selected_website_id': selected_website_id, 'page_obj': page_obj, 'contents_by_website': contents_by_website, 'page_size': page_size, 'recent_tasks': recent_tasks, 'total_media_files': total_media_files, } return render(request, 'crawler/dashboard.html', {'stats': stats}) def search_page(request): """搜索页面视图""" keyword = request.GET.get('q', '').strip() contents = [] if keyword: # 记录搜索关键字 SearchKeyword.objects.get_or_create( keyword=keyword, defaults={'last_used': timezone.now()} ) # 搜索内容 contents = CrawledContent.objects.filter( Q(title__icontains=keyword) | Q(content__icontains=keyword) | Q(keywords_matched__icontains=keyword) ).order_by('-created_at')[:50] return render(request, 'crawler/search.html', { 'keyword': keyword, 'contents': contents }) def preview_crawled_content(request, content_id): """预览爬取的内容""" content = get_object_or_404(CrawledContent, id=content_id) # 获取媒体文件 media_files = content.media_files.all() # 生成媒体文件HTML media_section = "" if media_files: media_section = """

媒体文件

""" for media_file in media_files: if media_file.media_type == 'image': media_section += f"""

图片: {media_file.alt_text or '无标题'}

{media_file.alt_text}

原始URL: {media_file.original_url}

文件大小: {media_file.file_size_display}

""" elif media_file.media_type == 'video': media_section += f"""

视频

原始URL: {media_file.original_url}

文件大小: {media_file.file_size_display}

""" elif media_file.media_type == 'audio': media_section += f"""

音频

原始URL: {media_file.original_url}

文件大小: {media_file.file_size_display}

""" else: media_section += f"""

文件: {media_file.get_media_type_display()}

下载文件

原始URL: {media_file.original_url}

文件大小: {media_file.file_size_display}

""" media_section += "
" # 处理内容格式，将换行符转换为段落和
标签 formatted_content = content.content.replace('\n\n', '

').replace('\n', '
') # 动态生成预览页面 html_content = f""" {content.title}

仪表板 管理界面 ← 返回

{content.title}

来源网站: {content.website.name} ({content.website.region})

原始链接: {content.url}

发布时间: {content.publish_date or '未知'}

作者: {content.author or '未知'}

匹配关键字: {content.keywords_matched}

爬取时间: {content.created_at}

媒体文件数量: {len(media_files)}

{formatted_content}

{media_section} """ return HttpResponse(html_content, content_type='text/html; charset=utf-8') def download_crawled_content(request, content_id): """下载文章内容为压缩包（包含Word文档和媒体文件）""" content = get_object_or_404(CrawledContent, id=content_id) # 创建内存中的字节流用于存储zip文件 zip_buffer = BytesIO() # 创建zip文件 with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: # 创建Word文档 doc = Document() doc.add_heading(content.title, 0) # 添加元数据 doc.add_paragraph(f'来源网站: {content.website.name} ({content.website.region})') doc.add_paragraph(f'原始链接: {content.url}') doc.add_paragraph(f'发布时间: {content.publish_date or "未知"}') doc.add_paragraph(f'作者: {content.author or "未知"}') doc.add_paragraph(f'匹配关键字: {content.keywords_matched}') doc.add_paragraph(f'爬取时间: {content.created_at}') # 添加内容 doc.add_heading('正文', level=1) for paragraph in content.content.split('\n\n'): if paragraph.strip(): doc.add_paragraph(paragraph.strip()) # 保存Word文档到内存 doc_buffer = BytesIO() doc.save(doc_buffer) doc_buffer.seek(0) # 添加Word文档到zip文件 zip_file.writestr(f"{content.title[:50]}.docx", doc_buffer.getvalue()) # 添加媒体文件到zip文件 media_files = content.media_files.all() for media_file in media_files: try: # 获取媒体文件的本地路径 if media_file.local_file and default_storage.exists(media_file.local_file.name): # 读取文件内容 file_content = default_storage.open(media_file.local_file.name).read() # 添加到zip文件中 zip_file.writestr(f"media/{os.path.basename(media_file.local_file.name)}", file_content) except Exception as e: # 如果文件无法读取，记录错误但继续处理其他文件 pass # 准备响应 zip_buffer.seek(0) response = HttpResponse(zip_buffer.getvalue(), content_type='application/zip') filename = f"{content.title[:50]}.zip" response['Content-Disposition'] = f'attachment; filename="{filename}"' return response def download_selected_contents(request): """下载选中的多篇文章内容为一个压缩包""" if request.method == 'POST': # 获取选中的文章ID selected_ids = request.POST.getlist('selected_contents') if not selected_ids: # 如果没有选中任何文章，返回错误 return HttpResponse("请至少选择一篇文章", status=400) # 获取选中的文章 contents = CrawledContent.objects.filter(id__in=selected_ids) if not contents.exists(): return HttpResponse("未找到选中的文章", status=404) # 创建内存中的字节流用于存储zip文件 zip_buffer = BytesIO() # 创建zip文件 with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: for content in contents: # 为每篇文章创建一个文件夹 folder_name = f"{content.title[:30].strip()}" # 确保文件夹名称合法 folder_name = "".join(c for c in folder_name if c.isalnum() or c in (' ', '-', '_')).rstrip() # 创建Word文档 doc = Document() doc.add_heading(content.title, 0) # 添加元数据 doc.add_paragraph(f'来源网站: {content.website.name} ({content.website.region})') doc.add_paragraph(f'原始链接: {content.url}') doc.add_paragraph(f'发布时间: {content.publish_date or "未知"}') doc.add_paragraph(f'作者: {content.author or "未知"}') doc.add_paragraph(f'匹配关键字: {content.keywords_matched}') doc.add_paragraph(f'爬取时间: {content.created_at}') # 添加内容 doc.add_heading('正文', level=1) for paragraph in content.content.split('\n\n'): if paragraph.strip(): doc.add_paragraph(paragraph.strip()) # 保存Word文档到内存 doc_buffer = BytesIO() doc.save(doc_buffer) doc_buffer.seek(0) # 添加Word文档到zip文件 zip_file.writestr(f"{folder_name}/{content.title[:50]}.docx", doc_buffer.getvalue()) # 添加媒体文件到zip文件 media_files = content.media_files.all() for media_file in media_files: try: # 获取媒体文件的本地路径 if media_file.local_file and default_storage.exists(media_file.local_file.name): # 读取文件内容 file_content = default_storage.open(media_file.local_file.name).read() # 添加到zip文件中 zip_file.writestr(f"{folder_name}/media/{os.path.basename(media_file.local_file.name)}", file_content) except Exception as e: # 如果文件无法读取，记录错误但继续处理其他文件 pass # 准备响应 zip_buffer.seek(0) response = HttpResponse(zip_buffer.getvalue(), content_type='application/zip') response['Content-Disposition'] = 'attachment; filename="selected_articles.zip"' return response return HttpResponse("无效的请求方法", status=405)