from django.db import models from django.utils import timezone import json class Website(models.Model): name = models.CharField(max_length=100, unique=True) base_url = models.URLField() description = models.TextField(blank=True, null=True) article_list_url = models.URLField(blank=True, null=True) article_selector = models.CharField(max_length=255, blank=True, null=True) content_selector = models.CharField(max_length=255, blank=True, null=True) enabled = models.BooleanField(default=True) def __str__(self): return self.name class Article(models.Model): website = models.ForeignKey(Website, on_delete=models.CASCADE) title = models.CharField(max_length=300) url = models.URLField(unique=True) pub_date = models.DateTimeField(null=True, blank=True) content = models.TextField() # html内容 created_at = models.DateTimeField(auto_now_add=True) media_files = models.JSONField(default=list, blank=True) def __str__(self): return self.title class CrawlTask(models.Model): """爬取任务模型""" TASK_STATUS_CHOICES = [ ('pending', '等待中'), ('running', '运行中'), ('completed', '已完成'), ('failed', '失败'), ('cancelled', '已取消'), ] TASK_TYPE_CHOICES = [ ('keyword', '关键词搜索'), ('historical', '历史文章'), ('full_site', '全站爬取'), ] name = models.CharField(max_length=200, verbose_name="任务名称") task_type = models.CharField(max_length=20, choices=TASK_TYPE_CHOICES, default='keyword', verbose_name="任务类型") keyword = models.CharField(max_length=200, blank=True, null=True, verbose_name="搜索关键词") websites = models.ManyToManyField(Website, blank=True, verbose_name="目标网站") start_date = models.DateField(blank=True, null=True, verbose_name="开始日期") end_date = models.DateField(blank=True, null=True, verbose_name="结束日期") max_pages = models.IntegerField(default=10, verbose_name="最大页数") max_articles = models.IntegerField(default=100, verbose_name="最大文章数") status = models.CharField(max_length=20, choices=TASK_STATUS_CHOICES, default='pending', verbose_name="状态") progress = models.IntegerField(default=0, verbose_name="进度百分比") current_website = models.CharField(max_length=100, blank=True, null=True, verbose_name="当前网站") current_action = models.CharField(max_length=200, blank=True, null=True, verbose_name="当前操作") total_articles = models.IntegerField(default=0, verbose_name="总文章数") success_count = models.IntegerField(default=0, verbose_name="成功数") failed_count = models.IntegerField(default=0, verbose_name="失败数") created_at = models.DateTimeField(auto_now_add=True, verbose_name="创建时间") started_at = models.DateTimeField(blank=True, null=True, verbose_name="开始时间") completed_at = models.DateTimeField(blank=True, null=True, verbose_name="完成时间") error_message = models.TextField(blank=True, null=True, verbose_name="错误信息") result_details = models.JSONField(default=dict, blank=True, verbose_name="结果详情") created_by = models.CharField(max_length=100, blank=True, null=True, verbose_name="创建者") class Meta: verbose_name = "爬取任务" verbose_name_plural = "爬取任务" ordering = ['-created_at'] def __str__(self): return f"{self.name} ({self.get_status_display()})" def get_websites_display(self): """获取网站列表的显示文本""" try: websites = self.websites.all() if not websites: return "所有网站" # 确保网站名称是字符串并可以被join处理 website_names = [str(w.name) for w in websites if w.name] return ", ".join(website_names) if website_names else "所有网站" except Exception: # 如果出现任何异常,返回默认值 return "所有网站" def get_duration(self): """获取任务执行时长""" if not self.started_at: return None end_time = self.completed_at or timezone.now() return end_time - self.started_at def is_running(self): """判断任务是否正在运行""" return self.status == 'running' def can_cancel(self): """判断任务是否可以取消""" return self.status in ['pending', 'running'] def get_progress_display(self): """获取进度显示文本""" if self.status == 'pending': return "等待开始" elif self.status == 'running': if self.current_website and self.current_action: return f"正在处理 {self.current_website}: {self.current_action}" return f"运行中 ({self.progress}%)" elif self.status == 'completed': return f"已完成 ({self.success_count}/{self.total_articles})" elif self.status == 'failed': return f"失败: {self.error_message[:50]}..." if self.error_message else "失败" elif self.status == 'cancelled': return "已取消" return "未知状态"