support keyword crawl

This commit is contained in:
2025-09-26 10:39:36 +08:00
parent 499454ff27
commit a36d730384
12 changed files with 2370 additions and 505 deletions

View File

@@ -462,9 +462,10 @@ class CrawlTaskAdmin(admin.ModelAdmin):
'status', 'progress', 'current_website', 'current_action',
'total_articles', 'success_count', 'failed_count',
'created_at', 'started_at', 'completed_at', 'error_message',
'result_details', 'duration_display', 'progress_display'
'result_details', 'duration_display', 'progress_display',
'execution_count', 'last_execution_at', 'execution_summary'
]
actions = ['start_tasks', 'cancel_tasks', 'delete_completed_tasks']
actions = ['start_tasks', 'rerun_tasks', 'cancel_tasks', 'delete_completed_tasks']
class Media:
js = ('admin/js/crawl_task_actions.js',)
@@ -488,6 +489,10 @@ class CrawlTaskAdmin(admin.ModelAdmin):
'fields': ('created_at', 'started_at', 'completed_at', 'duration_display'),
'classes': ('collapse',)
}),
('执行历史', {
'fields': ('execution_count', 'last_execution_at', 'execution_summary'),
'classes': ('collapse',)
}),
('错误信息', {
'fields': ('error_message',),
'classes': ('collapse',)
@@ -539,6 +544,11 @@ class CrawlTaskAdmin(admin.ModelAdmin):
return "-"
duration_display.short_description = '执行时长'
def execution_summary(self, obj):
"""执行摘要显示"""
return obj.get_execution_summary()
execution_summary.short_description = '执行摘要'
def actions_column(self, obj):
"""操作列"""
actions = []
@@ -551,6 +561,10 @@ class CrawlTaskAdmin(admin.ModelAdmin):
if obj.status == 'completed':
actions.append(f'<a href="javascript:void(0)" onclick="viewResults({obj.id})" class="button">查看结果</a>')
actions.append(f'<a href="javascript:void(0)" onclick="rerunTask({obj.id})" class="button" style="background-color: #28a745;">重新执行</a>')
if obj.status in ['failed', 'cancelled']:
actions.append(f'<a href="javascript:void(0)" onclick="rerunTask({obj.id})" class="button" style="background-color: #28a745;">重新执行</a>')
return format_html(' '.join(actions))
actions_column.short_description = '操作'
@@ -572,6 +586,23 @@ class CrawlTaskAdmin(admin.ModelAdmin):
self.message_user(request, f'成功启动 {started_count} 个任务', messages.SUCCESS)
start_tasks.short_description = '启动选中的任务'
def rerun_tasks(self, request, queryset):
"""重新执行选中的任务"""
rerun_count = 0
for task in queryset.filter(status__in=['completed', 'failed', 'cancelled']):
try:
success, message = task_executor.rerun_task(task.id)
if success:
rerun_count += 1
else:
self.message_user(request, f'重新执行任务 {task.name} 失败: {message}', messages.ERROR)
except Exception as e:
self.message_user(request, f'重新执行任务 {task.name} 失败: {e}', messages.ERROR)
if rerun_count > 0:
self.message_user(request, f'成功重新执行 {rerun_count} 个任务', messages.SUCCESS)
rerun_tasks.short_description = '重新执行选中的任务'
def cancel_tasks(self, request, queryset):
"""取消选中的任务"""
cancelled_count = 0
@@ -587,6 +618,9 @@ class CrawlTaskAdmin(admin.ModelAdmin):
if cancelled_count > 0:
self.message_user(request, f'成功取消 {cancelled_count} 个任务', messages.SUCCESS)
elif queryset.filter(status__in=['pending', 'running']).count() > 0:
# 有任务但没有成功取消任何任务
self.message_user(request, '没有成功取消任何任务', messages.WARNING)
cancel_tasks.short_description = '取消选中的任务'
def delete_completed_tasks(self, request, queryset):
@@ -628,6 +662,11 @@ class CrawlTaskAdmin(admin.ModelAdmin):
self.admin_site.admin_view(self.cancel_task_view),
name='cancel_task',
),
path(
'<int:task_id>/rerun/',
self.admin_site.admin_view(self.rerun_task_view),
name='rerun_task',
),
path(
'<int:task_id>/results/',
self.admin_site.admin_view(self.view_results_view),
@@ -640,7 +679,7 @@ class CrawlTaskAdmin(admin.ModelAdmin):
"""创建关键词搜索任务视图"""
if request.method == 'POST':
try:
from .utils import WEBSITE_SEARCH_CONFIGS
from .utils import WEBSITE_CRAWL_CONFIGS
name = request.POST.get('name', '')
keyword = request.POST.get('keyword', '')
@@ -688,7 +727,7 @@ class CrawlTaskAdmin(admin.ModelAdmin):
"""创建历史文章任务视图"""
if request.method == 'POST':
try:
from .utils import WEBSITE_SEARCH_CONFIGS
from .utils import WEBSITE_CRAWL_CONFIGS
name = request.POST.get('name', '')
websites = request.POST.getlist('websites')
@@ -733,7 +772,7 @@ class CrawlTaskAdmin(admin.ModelAdmin):
"""创建全站爬取任务视图"""
if request.method == 'POST':
try:
from .utils import WEBSITE_SEARCH_CONFIGS
from .utils import WEBSITE_CRAWL_CONFIGS
name = request.POST.get('name', '')
websites = request.POST.getlist('websites')
@@ -783,6 +822,19 @@ class CrawlTaskAdmin(admin.ModelAdmin):
return HttpResponseRedirect(reverse('admin:core_crawltask_changelist'))
def rerun_task_view(self, request, task_id):
"""重新执行任务视图"""
try:
success, message = task_executor.rerun_task(task_id)
if success:
self.message_user(request, f'任务已重新执行: {message}', messages.SUCCESS)
else:
self.message_user(request, f'重新执行任务失败: {message}', messages.ERROR)
except Exception as e:
self.message_user(request, f'重新执行任务失败: {e}', messages.ERROR)
return HttpResponseRedirect(reverse('admin:core_crawltask_changelist'))
def cancel_task_view(self, request, task_id):
"""取消任务视图"""
try: