Add Support All Platform

core/templates/core/article_detail.html

@@ -77,7 +77,8 @@
         <div class="meta">
             网站: {{ article.website.name }} |
             发布时间: {{ article.pub_date|date:"Y-m-d H:i" }} |
-            创建时间: {{ article.created_at|date:"Y-m-d H:i" }}
+            创建时间: {{ article.created_at|date:"Y-m-d H:i" }} |
+            源网址: <a href="{{ article.url }}" target="_blank">{{ article.url }}</a>
         </div>
 
         <div class="content">

core/utils.py

@@ -126,7 +126,8 @@ def process_article(url, website):
                 soup.find("div", class_="content") or
                 soup.find("div", id="content") or
                 soup.find("div", class_="article") or
-                soup.find("div", class_="main-content")
+                soup.find("div", class_="main-content") or
+                soup.find("span", id="detailContent")  # 添加新华网特有的内容容器
         )
     elif website.name == "东方烟草报":
         # 优化东方烟草报的标题提取逻辑，按优先级尝试多种选择器
@@ -445,7 +446,8 @@ def process_article(url, website):
                 paging_element.decompose()
 
             # 移除政协号客户端下载提示
-            for zxh_element in content_tag.find_all("div", style=lambda x: x and "background:#F9F9F9;padding:50px" in x):
+            for zxh_element in content_tag.find_all("div",
+                                                    style=lambda x: x and "background:#F9F9F9;padding:50px" in x):
                 zxh_element.decompose()
 
             # 移除版权信息
@@ -503,6 +505,7 @@ def process_article(url, website):
                 soup.find("title")
         )
         content_tag = (
+                soup.find("div", id="detail") or  # 添加学习时报特有内容容器
                 soup.find("div", class_="content") or
                 soup.find("div", class_="article-content") or
                 soup.find("div", id="content") or
@@ -511,6 +514,24 @@ def process_article(url, website):
                 soup.find("div", class_="article") or
                 soup.find("div", class_="article-body")
         )
+
+        # 针对学习时报的特殊处理，清理内容中的无关元素
+        if content_tag:
+            # 移除编辑信息
+            for editor_element in content_tag.find_all("div", class_="editor"):
+                editor_element.decompose()
+
+            # 移除分享相关元素
+            for share_element in content_tag.find_all("div", class_="share"):
+                share_element.decompose()
+
+            # 移除无关的TRS_Editor包装层
+            for trs_editor in content_tag.find_all("div", class_="TRS_Editor"):
+                trs_editor.unwrap()  # unwrap只移除标签，保留内容
+
+            # 移除Custom_UnionStyle包装层
+            for custom_style in content_tag.find_all("div", class_="Custom_UnionStyle"):
+                custom_style.unwrap()  # unwrap只移除标签，保留内容
     elif "中国青年报" in website.name or "cyol" in website.name:
         # 中国青年报的文章结构处理 - 修复无法爬取问题
         title_tag = (
@@ -532,6 +553,7 @@ def process_article(url, website):
         title_tag = (
                 soup.find("h1", class_="title") or
                 soup.find("h1") or
+                soup.find("p", class_="f_container_title") or  # 添加中国妇女报特有标题容器
                 soup.find("title")
         )
         content_tag = (
@@ -541,7 +563,9 @@ def process_article(url, website):
                 soup.find("div", class_="text") or
                 soup.find("div", class_="main-content") or
                 soup.find("div", class_="article") or
-                soup.find("div", class_="article-body")
+                soup.find("div", class_="article-body") or
+                soup.find("div", class_="f_container_left") or  # 添加中国妇女报特有内容容器
+                soup.find("div", class_="f_container")  # 添加另一种可能的内容容器
         )
     elif "法治日报" in website.name or "legaldaily" in website.name:
         # 法治日报的文章结构处理 - 修复无法爬取问题
@@ -714,6 +738,28 @@ def process_article(url, website):
         if not src:
             src = video.get("data-url") or video.get("data-video")
 
+        # 新增：检查新华网特有的视频源属性
+        if not src:
+            src = video.get("data-video-src")
+
+        # 新增：针对新华网的特殊处理，从复杂播放器结构中提取视频源
+        if not src and "新华网" in website.name:
+            # 尝试从video标签的属性中直接获取src
+            for attr in video.attrs:
+                if 'src' in attr.lower():
+                    src = video.attrs.get(attr)
+                    break
+
+            # 如果还是没有找到，尝试查找父容器中的视频源信息
+            if not src:
+                parent = video.parent
+                if parent and parent.name == 'div' and 'player-container' in parent.get('class', []):
+                    # 检查是否有data-*属性包含视频信息
+                    for attr, value in parent.attrs.items():
+                        if 'data' in attr and isinstance(value, str) and ('.mp4' in value or 'video' in value):
+                            src = value
+                            break
+
         if not src:
             continue
 
@@ -726,6 +772,10 @@ def process_article(url, website):
             if "cctv.com" in src or "cntv.cn" in src:
                 print(f"发现央视视频: {src}")
 
+        # 针对新华网的特殊处理
+        elif "新华网" in website.name:
+            print(f"发现新华网视频: {src}")
+
         local_path = download_media(src, save_dir)
         if local_path:
             rel_path = os.path.relpath(local_path, settings.MEDIA_ROOT)
@@ -812,6 +862,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
                     soup.find("div", id="content") is not None or
                     soup.find("div", class_="article") is not None or
                     soup.find("div", class_="main-content") is not None or
+                    soup.find("span", id="detailContent") is not None or  # 添加新华网特有内容容器判断
                     ("/news/" in path) or
                     ("/article/" in path) or
                     (path.startswith("/detail/") and len(path) > 10)
@@ -1064,6 +1115,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
                     soup.find("div", class_="main-content") is not None or
                     soup.find("div", class_="article") is not None or
                     soup.find("div", class_="article-body") is not None or
+                    soup.find("div", id="detail") is not None or  # 添加学习时报特有内容容器判断
                     ("/article/" in path) or
                     ("/content/" in path) or
                     (path.startswith("/detail/") and len(path) > 10)
@@ -1168,7 +1220,6 @@ def full_site_crawler(start_url, website, max_pages=1000):
                     soup.find("div", class_="article-content") is not None or
                     (soup.find("div", id="content") is not None and
                      soup.find("h1") is not None) or
-                    soup.find("div", class_="text") is not None or
                     soup.find("div", class_="main-content") is not None or
                     soup.find("div", class_="article") is not None or
                     soup.find("div", class_="article-body") is not None or
@@ -1220,7 +1271,7 @@ def full_site_crawler(start_url, website, max_pages=1000):
                     if ("/article/" in href_path or
                             href_path.startswith("/detail/") or
                             ("/dynamic/" in href_path and "article" in href_path) or
-                            href_path.count("/") > 2):  # 更深层的页面可能是文章页
+                            href_path.count("/") > 2):  # 更深层
                         queue.append(href)
             elif href not in visited and is_valid_url(href, base_netloc):
                 queue.append(href)