fix bugs and support all platform

This commit is contained in:
2025-08-15 08:33:47 +08:00
parent e82b85f4dd
commit 4945b4c6b0
36 changed files with 2296 additions and 992 deletions

View File

@@ -3,6 +3,7 @@
{% block object-tools %}
{{ block.super }}
<!--
<div style="margin-top: 10px;">
<form method="post" action="{% url 'admin:run_crawler' %}" style="display: inline-block;">
{% csrf_token %}
@@ -16,4 +17,5 @@
<input type="submit" value="执行爬虫" class="default" style="margin-left: 10px;"/>
</form>
</div>
-->
{% endblock %}

View File

@@ -0,0 +1,304 @@
{% extends "admin/base_site.html" %}
{% load static %}
{% block title %}爬虫状态 - {{ site_title|default:_('Django site admin') }}{% endblock %}
{% block extrastyle %}
<style>
.status-card {
background: white;
border: 1px solid #ddd;
border-radius: 8px;
padding: 20px;
margin: 20px 0;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
}
.status-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
padding-bottom: 10px;
border-bottom: 2px solid #f0f0f0;
}
.status-title {
font-size: 24px;
font-weight: bold;
color: #333;
}
.stats-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 20px;
margin-bottom: 30px;
}
.stat-card {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 20px;
border-radius: 8px;
text-align: center;
}
.stat-number {
font-size: 32px;
font-weight: bold;
margin-bottom: 5px;
}
.stat-label {
font-size: 14px;
opacity: 0.9;
}
.nodes-section, .batches-section {
margin-top: 30px;
}
.section-title {
font-size: 20px;
font-weight: bold;
margin-bottom: 15px;
color: #333;
}
.node-item, .batch-item {
background: #f8f9fa;
border: 1px solid #e9ecef;
border-radius: 6px;
padding: 15px;
margin-bottom: 10px;
}
.node-header, .batch-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 10px;
}
.node-name, .batch-id {
font-weight: bold;
color: #333;
}
.node-status, .batch-status {
padding: 4px 8px;
border-radius: 4px;
font-size: 12px;
font-weight: bold;
}
.status-active {
background: #d4edda;
color: #155724;
}
.status-running {
background: #fff3cd;
color: #856404;
}
.status-completed {
background: #d1ecf1;
color: #0c5460;
}
.status-failed {
background: #f8d7da;
color: #721c24;
}
.node-details, .batch-details {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
gap: 10px;
font-size: 14px;
}
.detail-item {
display: flex;
justify-content: space-between;
}
.detail-label {
color: #666;
}
.detail-value {
font-weight: bold;
color: #333;
}
.progress-bar {
width: 100%;
height: 8px;
background: #e9ecef;
border-radius: 4px;
overflow: hidden;
margin-top: 10px;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #28a745, #20c997);
transition: width 0.3s ease;
}
.refresh-btn {
background: #007bff;
color: white;
border: none;
padding: 8px 16px;
border-radius: 4px;
cursor: pointer;
font-size: 14px;
}
.refresh-btn:hover {
background: #0056b3;
}
.no-data {
text-align: center;
color: #666;
padding: 40px;
font-style: italic;
}
</style>
{% endblock %}
{% block content %}
<div class="status-card">
<div class="status-header">
<h1 class="status-title">爬虫状态监控</h1>
<button class="refresh-btn" onclick="location.reload()">刷新</button>
</div>
<!-- 统计卡片 -->
<div class="stats-grid">
<div class="stat-card">
<div class="stat-number">{{ task_stats.total_nodes }}</div>
<div class="stat-label">活跃节点</div>
</div>
<div class="stat-card">
<div class="stat-number">{{ task_stats.active_tasks }}</div>
<div class="stat-label">运行中任务</div>
</div>
<div class="stat-card">
<div class="stat-number">{{ task_stats.total_batches }}</div>
<div class="stat-label">总批次</div>
</div>
<div class="stat-card">
<div class="stat-number">{{ nodes|length }}</div>
<div class="stat-label">在线节点</div>
</div>
</div>
<!-- 节点状态 -->
<div class="nodes-section">
<h2 class="section-title">爬虫节点状态</h2>
{% if nodes %}
{% for node in nodes %}
<div class="node-item">
<div class="node-header">
<span class="node-name">{{ node.node_id }}</span>
<span class="node-status status-active">{{ node.status }}</span>
</div>
<div class="node-details">
<div class="detail-item">
<span class="detail-label">活跃任务:</span>
<span class="detail-value">{{ node.active_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">完成任务:</span>
<span class="detail-value">{{ node.completed_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">失败任务:</span>
<span class="detail-value">{{ node.failed_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">最后心跳:</span>
<span class="detail-value">
{% if node.last_heartbeat %}
{{ node.last_heartbeat|date:"H:i:s" }}
{% else %}
未知
{% endif %}
</span>
</div>
</div>
</div>
{% endfor %}
{% else %}
<div class="no-data">
暂无活跃的爬虫节点
</div>
{% endif %}
</div>
<!-- 批次状态 -->
<div class="batches-section">
<h2 class="section-title">最近批次</h2>
{% if batches %}
{% for batch in batches %}
<div class="batch-item">
<div class="batch-header">
<span class="batch-id">{{ batch.batch_id }}</span>
<span class="batch-status status-{{ batch.status }}">
{% if batch.status == 'running' %}
运行中
{% elif batch.status == 'completed' %}
已完成
{% elif batch.status == 'failed' %}
失败
{% else %}
{{ batch.status }}
{% endif %}
</span>
</div>
<div class="batch-details">
<div class="detail-item">
<span class="detail-label">总任务:</span>
<span class="detail-value">{{ batch.total_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">已完成:</span>
<span class="detail-value">{{ batch.completed_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">失败:</span>
<span class="detail-value">{{ batch.failed_tasks }}</span>
</div>
<div class="detail-item">
<span class="detail-label">进度:</span>
<span class="detail-value">{{ batch.progress|floatformat:1 }}%</span>
</div>
</div>
{% if batch.status == 'running' %}
<div class="progress-bar">
<div class="progress-fill" style="width: {{ batch.progress }}%"></div>
</div>
{% endif %}
</div>
{% endfor %}
{% else %}
<div class="no-data">
暂无批次记录
</div>
{% endif %}
</div>
</div>
<script>
// 自动刷新页面
setTimeout(function () {
location.reload();
}, 30000); // 30秒刷新一次
</script>
{% endblock %}

View File

@@ -40,7 +40,16 @@
margin-top: 20px;
}
.content img {
/* 优化:确保图片和视频不会超出容器显示 */
.content img, .content video {
max-width: 100%;
height: auto;
display: block;
margin: 10px 0;
}
/* 优化:确保iframe也不会超出容器显示 */
.content iframe {
max-width: 100%;
height: auto;
}
@@ -61,7 +70,7 @@
body {
padding: 10px;
}
.container {
padding: 15px;
}
@@ -69,21 +78,21 @@
</style>
</head>
<body>
<div class="container">
<a href="{% url 'article_list' %}" class="back-link">&laquo; 返回文章列表</a>
<div class="container">
<a href="{% url 'article_list' %}" class="back-link">&laquo; 返回文章列表</a>
<h1>{{ article.title }}</h1>
<h1>{{ article.title }}</h1>
<div class="meta">
网站: {{ article.website.name }} |
发布时间: {{ article.pub_date|date:"Y-m-d H:i" }} |
创建时间: {{ article.created_at|date:"Y-m-d H:i" }} |
源网址: <a href="{{ article.url }}" target="_blank">{{ article.url }}</a>
</div>
<div class="content">
{{ article.content|safe }}
</div>
<div class="meta">
网站: {{ article.website.name }} |
发布时间: {{ article.pub_date|date:"Y-m-d H:i" }} |
创建时间: {{ article.created_at|date:"Y-m-d H:i" }} |
源网址: <a href="{{ article.url }}" target="_blank">{{ article.url }}</a>
</div>
<div class="content">
{{ article.content|safe }}
</div>
</div>
</body>
</html>

View File

@@ -17,7 +17,7 @@
background: white;
padding: 30px;
margin-bottom: 20px;
box-shadow: 0 2px 5px rgba(0,0,0,0.05); /* 添加轻微阴影 */
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05); /* 添加轻微阴影 */
border-radius: 8px; /* 添加圆角 */
}
@@ -240,7 +240,7 @@
<form method="get">
<input type="text" name="q" placeholder="输入关键词搜索文章..." value="{{ search_query }}">
{% if selected_website %}
<input type="hidden" name="website" value="{{ selected_website.id }}">
<input type="hidden" name="website" value="{{ selected_website.id }}">
{% endif %}
<input type="submit" value="搜索">
</form>
@@ -251,9 +251,11 @@
<div class="sidebar">
<div class="filters">
<strong>按网站筛选:</strong>
<a href="{% url 'article_list' %}{% if search_query %}?q={{ search_query }}{% endif %}" {% if not selected_website %}class="active" {% endif %}>全部</a>
<a href="{% url 'article_list' %}{% if search_query %}?q={{ search_query }}{% endif %}"
{% if not selected_website %}class="active" {% endif %}>全部</a>
{% for website in websites %}
<a href="?website={{ website.id }}{% if search_query %}&q={{ search_query }}{% endif %}" {% if selected_website and selected_website.id == website.id %}class="active" {% endif %}>{{ website.name }}</a>
<a href="?website={{ website.id }}{% if search_query %}&q={{ search_query }}{% endif %}"
{% if selected_website and selected_website.id == website.id %}class="active" {% endif %}>{{ website.name }}</a>
{% endfor %}
</div>
</div>
@@ -262,10 +264,10 @@
<div class="main-content">
<!-- 新增:搜索结果信息 -->
{% if search_query %}
<div class="search-info">
搜索 "{{ search_query }}" 找到 {{ page_obj.paginator.count }} 篇文章
<a href="{% if selected_website %}?website={{ selected_website.id }}{% else %}{% url 'article_list' %}{% endif %}">清除搜索</a>
</div>
<div class="search-info">
搜索 "{{ search_query }}" 找到 {{ page_obj.paginator.count }} 篇文章
<a href="{% if selected_website %}?website={{ selected_website.id }}{% else %}{% url 'article_list' %}{% endif %}">清除搜索</a>
</div>
{% endif %}
<!-- 新增:导出功能 -->
@@ -280,60 +282,70 @@
<ul>
{% for article in page_obj %}
<li>
<input type="checkbox" class="article-checkbox" value="{{ article.id }}" id="article_{{ article.id }}">
<a href="{% url 'article_detail' article.id %}">{{ article.title }}</a>
<div class="meta">({{ article.website.name }} - {{ article.created_at|date:"Y-m-d" }})</div>
</li>
{% empty %}
<li>暂无文章</li>
<li>
<input type="checkbox" class="article-checkbox" value="{{ article.id }}"
id="article_{{ article.id }}">
<a href="{% url 'article_detail' article.id %}">{{ article.title }}</a>
<div class="meta">({{ article.website.name }} - {{ article.created_at|date:"Y-m-d" }})</div>
</li>
{% empty %}
<li>暂无文章</li>
{% endfor %}
</ul>
<div class="pagination">
{% if page_obj.has_previous %}
{% if selected_website %}
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page=1">&laquo; 首页</a>
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.previous_page_number }}">上一页</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page=1">&laquo; 首页</a>
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.previous_page_number }}">上一页</a>
{% endif %}
{% if selected_website %}
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page=1">&laquo;
首页</a>
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.previous_page_number }}">上一页</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page=1">&laquo; 首页</a>
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.previous_page_number }}">上一页</a>
{% endif %}
{% endif %}
<span>第 {{ page_obj.number }} 页,共 {{ page_obj.paginator.num_pages }} 页</span>
<!-- 修改:优化页码显示逻辑 -->
{% with page_obj.paginator as paginator %}
{% for num in paginator.page_range %}
{% if page_obj.number == num %}
<a href="#" class="current">{{ num }}</a>
{% elif num > page_obj.number|add:'-3' and num < page_obj.number|add:'3' %}
{% if selected_website %}
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ num }}">{{ num }}</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ num }}">{{ num }}</a>
{% endif %}
{% elif num == 1 or num == paginator.num_pages %}
{% if selected_website %}
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ num }}">{{ num }}</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ num }}">{{ num }}</a>
{% endif %}
{% elif num == page_obj.number|add:'-3' or num == page_obj.number|add:'3' %}
<span class="ellipsis">...</span>
{% endif %}
{% endfor %}
{% for num in paginator.page_range %}
{% if page_obj.number == num %}
<a href="#" class="current">{{ num }}</a>
{% elif num > page_obj.number|add:'-3' and num < page_obj.number|add:'3' %}
{% if selected_website %}
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ num }}">{{ num }}</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ num }}">{{ num }}</a>
{% endif %}
{% elif num == 1 or num == paginator.num_pages %}
{% if selected_website %}
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ num }}">{{ num }}</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ num }}">{{ num }}</a>
{% endif %}
{% elif num == page_obj.number|add:'-3' or num == page_obj.number|add:'3' %}
<span class="ellipsis">...</span>
{% endif %}
{% endfor %}
{% endwith %}
{% if page_obj.has_next %}
{% if selected_website %}
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.next_page_number }}">下一页</a>
<a href="?website={{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.paginator.num_pages }}">末页 &raquo;</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.next_page_number }}">下一页</a>
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.paginator.num_pages }}">末页 &raquo;</a>
{% endif %}
{% if selected_website %}
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.next_page_number }}">下一页</a>
<a href="?website=
{{ selected_website.id }}{% if search_query %}&q={{ search_query }}{% endif %}&page={{ page_obj.paginator.num_pages }}">末页
&raquo;</a>
{% else %}
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.next_page_number }}">下一页</a>
<a href="?{% if search_query %}q={{ search_query }}&{% endif %}page={{ page_obj.paginator.num_pages }}">末页
&raquo;</a>
{% endif %}
{% endif %}
</div>
</div>
@@ -396,25 +408,25 @@
format: 'json'
})
})
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.json';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.json';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
});
// 导出为CSV功能
@@ -434,25 +446,25 @@
format: 'csv'
})
})
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.csv';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.csv';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
});
// 新增:导出为ZIP包功能
@@ -472,25 +484,25 @@
format: 'zip' // 指定导出格式为ZIP
})
})
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.zip';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
.then(response => {
if (response.ok) {
return response.blob();
}
throw new Error('导出失败');
})
.then(blob => {
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'articles.zip';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
})
.catch(error => {
alert('导出失败: ' + error);
});
});
// 初始化导出按钮状态