nas-tools/app/rss.py
2023-02-14 11:25:02 +08:00

630 lines
29 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import re
import xml.dom.minidom
from threading import Lock
import log
from app.downloader import Downloader
from app.filter import Filter
from app.helper import DbHelper
from app.media import Media
from app.media.meta import MetaInfo
from app.sites import Sites
from app.subscribe import Subscribe
from app.utils import DomUtils, RequestUtils, StringUtils, ExceptionUtils, RssTitleUtils, Torrent
from app.utils.types import MediaType, SearchType
lock = Lock()
class Rss:
_sites = []
filter = None
media = None
sites = None
downloader = None
searcher = None
dbhelper = None
subscribe = None
def __init__(self):
self.init_config()
def init_config(self):
self.media = Media()
self.downloader = Downloader()
self.sites = Sites()
self.filter = Filter()
self.dbhelper = DbHelper()
self.subscribe = Subscribe()
self._sites = self.sites.get_sites(rss=True)
def rssdownload(self):
"""
RSS订阅检索下载入口由定时服务调用
"""
if not self._sites:
return
with lock:
log.info("【Rss】开始RSS订阅...")
# 读取电影订阅
rss_movies = self.subscribe.get_subscribe_movies(state='R')
if not rss_movies:
log.warn("【Rss】没有正在订阅的电影")
else:
log.info("【Rss】电影订阅清单%s"
% " ".join('%s' % info.get("name") for _, info in rss_movies.items()))
# 读取电视剧订阅
rss_tvs = self.subscribe.get_subscribe_tvs(state='R')
if not rss_tvs:
log.warn("【Rss】没有正在订阅的电视剧")
else:
log.info("【Rss】电视剧订阅清单%s"
% " ".join('%s' % info.get("name") for _, info in rss_tvs.items()))
# 没有订阅退出
if not rss_movies and not rss_tvs:
return
# 获取有订阅的站点范围
check_sites = []
check_all = False
for rid, rinfo in rss_movies.items():
rss_sites = rinfo.get("rss_sites")
if not rss_sites:
check_all = True
break
else:
check_sites += rss_sites
if not check_all:
for rid, rinfo in rss_tvs.items():
rss_sites = rinfo.get("rss_sites")
if not rss_sites:
check_all = True
break
else:
check_sites += rss_sites
if check_all:
check_sites = []
else:
check_sites = list(set(check_sites))
# 匹配到的资源列表
rss_download_torrents = []
# 缺失的资源详情
rss_no_exists = {}
# 遍历站点资源
for site_info in self._sites:
if not site_info:
continue
# 站点名称
site_name = site_info.get("name")
# 没有订阅的站点中的不检索
if check_sites and site_name not in check_sites:
continue
# 站点rss链接
rss_url = site_info.get("rssurl")
if not rss_url:
log.info(f"【Rss】{site_name} 未配置rssurl跳过...")
continue
site_cookie = site_info.get("cookie")
site_ua = site_info.get("ua")
# 是否解析种子详情
site_parse = site_info.get("parse")
# 是否使用代理
site_proxy = site_info.get("proxy")
# 使用的规则
site_fliter_rule = site_info.get("rule")
# 开始下载RSS
log.info(f"【Rss】正在处理{site_name}")
if site_info.get("pri"):
site_order = 100 - int(site_info.get("pri"))
else:
site_order = 0
rss_acticles = self.parse_rssxml(rss_url)
if not rss_acticles:
log.warn(f"【Rss】{site_name} 未下载到数据")
continue
else:
log.info(f"【Rss】{site_name} 获取数据:{len(rss_acticles)}")
# 处理RSS结果
res_num = 0
for article in rss_acticles:
try:
# 种子名
title = article.get('title')
# 种子链接
enclosure = article.get('enclosure')
# 种子页面
page_url = article.get('link')
# 种子大小
size = article.get('size')
# 开始处理
log.info(f"【Rss】开始处理{title}")
# 检查这个种子是不是下过了
if self.dbhelper.is_torrent_rssd(enclosure):
log.info(f"【Rss】{title} 已成功订阅过")
continue
# 识别种子名称开始检索TMDB
media_info = MetaInfo(title=title)
cache_info = self.media.get_cache_info(media_info)
if cache_info.get("id"):
# 使用缓存信息
media_info.tmdb_id = cache_info.get("id")
media_info.type = cache_info.get("type")
media_info.title = cache_info.get("title")
media_info.year = cache_info.get("year")
else:
# 重新查询TMDB
media_info = self.media.get_media_info(title=title)
if not media_info:
log.warn(f"【Rss】{title} 无法识别出媒体信息!")
continue
elif not media_info.tmdb_info:
log.info(f"【Rss】{title} 识别为 {media_info.get_name()} 未匹配到TMDB媒体信息")
# 大小及种子页面
media_info.set_torrent_info(size=size,
page_url=page_url,
site=site_name,
site_order=site_order,
enclosure=enclosure)
# 检查种子是否匹配订阅返回匹配到的订阅ID、是否洗版、总集数、上传因子、下载因子
match_flag, match_msg, match_info = self.check_torrent_rss(
media_info=media_info,
rss_movies=rss_movies,
rss_tvs=rss_tvs,
site_filter_rule=site_fliter_rule,
site_cookie=site_cookie,
site_parse=site_parse,
site_ua=site_ua,
site_proxy=site_proxy)
for msg in match_msg:
log.info(f"【Rss】{msg}")
# 未匹配
if not match_flag:
continue
# 非模糊匹配命中,检查本地情况,检查删除订阅
if not match_info.get("fuzzy_match"):
# 匹配到订阅如没有TMDB信息则重新查询
if not media_info.tmdb_info and media_info.tmdb_id:
media_info.set_tmdb_info(self.media.get_tmdb_info(mtype=media_info.type,
tmdbid=media_info.tmdb_id))
if not media_info.tmdb_info:
continue
# 非洗版时检查本地是否存在
if not match_info.get("over_edition"):
if media_info.type == MediaType.MOVIE:
exist_flag, rss_no_exists, _ = self.downloader.check_exists_medias(
meta_info=media_info,
no_exists=rss_no_exists
)
else:
# 从登记薄中获取缺失剧集
season = 1
if match_info.get("season"):
season = int(str(match_info.get("season")).replace("S", ""))
# 设定的总集数
total_ep = match_info.get("total")
# 设定的开始集数
current_ep = match_info.get("current_ep")
# 表登记的缺失集数
episodes = self.subscribe.get_subscribe_tv_episodes(match_info.get("id"))
if episodes is None:
episodes = []
if current_ep:
episodes = list(range(int(current_ep), int(total_ep) + 1))
rss_no_exists[media_info.tmdb_id] = [
{
"season": season,
"episodes": episodes,
"total_episodes": total_ep
}
]
else:
rss_no_exists[media_info.tmdb_id] = [
{
"season": season,
"episodes": episodes,
"total_episodes": total_ep
}
]
# 检查本地媒体库情况
exist_flag, library_no_exists, _ = self.downloader.check_exists_medias(
meta_info=media_info,
total_ep={season: total_ep}
)
# 取交集做为缺失集
rss_no_exists = Torrent.get_intersection_episodes(target=rss_no_exists,
source=library_no_exists,
title=media_info.tmdb_id)
if rss_no_exists.get(media_info.tmdb_id):
log.info("【Rss】%s 订阅缺失季集:%s" % (
media_info.get_title_string(),
rss_no_exists.get(media_info.tmdb_id)
))
# 本地已存在
if exist_flag:
continue
# 洗版模式
else:
# 洗版时季集不完整的资源不要
if media_info.type != MediaType.MOVIE \
and media_info.get_episode_list():
log.info(
f"【Rss】{media_info.get_title_string()}{media_info.get_season_string()} "
f"正在洗版,过滤掉季集不完整的资源:{title}"
)
continue
if not self.subscribe.check_subscribe_over_edition(
rtype=media_info.type,
rssid=match_info.get("id"),
res_order=match_info.get("res_order")):
log.info(
f"【Rss】{media_info.get_title_string()}{media_info.get_season_string()} "
f"正在洗版,跳过低优先级或同优先级资源:{title}"
)
continue
# 模糊匹配
else:
# 不做处理,直接下载
pass
# 设置种子信息
media_info.set_torrent_info(res_order=match_info.get("res_order"),
filter_rule=match_info.get("filter_rule"),
over_edition=match_info.get("over_edition"),
download_volume_factor=match_info.get("download_volume_factor"),
upload_volume_factor=match_info.get("upload_volume_factor"),
rssid=match_info.get("id"))
# 设置下载参数
media_info.set_download_info(download_setting=match_info.get("download_setting"),
save_path=match_info.get("save_path"))
# 插入数据库历史记录
self.dbhelper.insert_rss_torrents(media_info)
# 加入下载列表
if media_info not in rss_download_torrents:
rss_download_torrents.append(media_info)
res_num = res_num + 1
except Exception as e:
ExceptionUtils.exception_traceback(e)
log.error("【Rss】处理RSS发生错误%s" % str(e))
continue
log.info("【Rss】%s 处理结束,匹配到 %s 个有效资源" % (site_name, res_num))
log.info("【Rss】所有RSS处理结束%s 个有效资源" % len(rss_download_torrents))
# 开始择优下载
self.download_rss_torrent(rss_download_torrents=rss_download_torrents,
rss_no_exists=rss_no_exists)
@staticmethod
def parse_rssxml(url):
"""
解析RSS订阅URL获取RSS中的种子信息
:param url: RSS地址
:return: 种子信息列表
"""
_special_title_sites = {
'pt.keepfrds.com': RssTitleUtils.keepfriends_title
}
# 开始处理
ret_array = []
if not url:
return []
site_domain = StringUtils.get_url_domain(url)
try:
ret = RequestUtils().get_res(url)
if not ret:
return []
ret.encoding = ret.apparent_encoding
except Exception as e2:
ExceptionUtils.exception_traceback(e2)
log.console(str(e2))
return []
if ret:
ret_xml = ret.text
try:
# 解析XML
dom_tree = xml.dom.minidom.parseString(ret_xml)
rootNode = dom_tree.documentElement
items = rootNode.getElementsByTagName("item")
for item in items:
try:
# 标题
title = DomUtils.tag_value(item, "title", default="")
if not title:
continue
# 标题特殊处理
if site_domain and site_domain in _special_title_sites:
title = _special_title_sites.get(site_domain)(title)
# 描述
description = DomUtils.tag_value(item, "description", default="")
# 种子页面
link = DomUtils.tag_value(item, "link", default="")
# 种子链接
enclosure = DomUtils.tag_value(item, "enclosure", "url", default="")
if not enclosure and not link:
continue
# 部分RSS只有link没有enclosure
if not enclosure and link:
enclosure = link
link = None
# 大小
size = DomUtils.tag_value(item, "enclosure", "length", default=0)
if size and str(size).isdigit():
size = int(size)
else:
size = 0
# 发布日期
pubdate = DomUtils.tag_value(item, "pubDate", default="")
if pubdate:
# 转换为时间
pubdate = StringUtils.get_time_stamp(pubdate)
# 返回对象
tmp_dict = {'title': title,
'enclosure': enclosure,
'size': size,
'description': description,
'link': link,
'pubdate': pubdate}
ret_array.append(tmp_dict)
except Exception as e1:
ExceptionUtils.exception_traceback(e1)
continue
except Exception as e2:
ExceptionUtils.exception_traceback(e2)
return ret_array
return ret_array
def check_torrent_rss(self,
media_info,
rss_movies,
rss_tvs,
site_filter_rule,
site_cookie,
site_parse,
site_ua,
site_proxy):
"""
判断种子是否命中订阅
:param media_info: 已识别的种子媒体信息
:param rss_movies: 电影订阅清单
:param rss_tvs: 电视剧订阅清单
:param site_filter_rule: 站点过滤规则
:param site_cookie: 站点的Cookie
:param site_parse: 是否解析种子详情
:param site_ua: 站点请求UA
:param site_proxy: 是否使用代理
:return: 匹配到的订阅ID、是否洗版、总集数、匹配规则的资源顺序、上传因子、下载因子匹配的季电视剧
"""
# 默认值
# 匹配状态 0不在订阅范围内 -1不符合过滤条件 1匹配
match_flag = False
# 匹配的rss信息
match_msg = []
match_rss_info = {}
# 上传因素
upload_volume_factor = None
# 下载因素
download_volume_factor = None
hit_and_run = False
# 匹配电影
if media_info.type == MediaType.MOVIE and rss_movies:
for rid, rss_info in rss_movies.items():
rss_sites = rss_info.get('rss_sites')
# 过滤订阅站点
if rss_sites and media_info.site not in rss_sites:
continue
# tmdbid或名称年份匹配
name = rss_info.get('name')
year = rss_info.get('year')
tmdbid = rss_info.get('tmdbid')
fuzzy_match = rss_info.get('fuzzy_match')
# 非模糊匹配
if not fuzzy_match:
# 有tmdbid时使用tmdbid匹配
if tmdbid and not tmdbid.startswith("DB:"):
if str(media_info.tmdb_id) != str(tmdbid):
continue
else:
# 豆瓣年份与tmdb取向不同
if year and str(media_info.year) not in [str(year),
str(int(year) + 1),
str(int(year) - 1)]:
continue
if name != media_info.title:
continue
# 模糊匹配
else:
# 匹配年份
if year and str(year) != str(media_info.year):
continue
# 匹配关键字或正则表达式
search_title = f"{media_info.org_string} {media_info.title} {media_info.year}"
if not re.search(name, search_title, re.I) and name not in search_title:
continue
# 媒体匹配成功
match_flag = True
match_rss_info = rss_info
break
# 匹配电视剧
elif rss_tvs:
# 匹配种子标题
for rid, rss_info in rss_tvs.items():
rss_sites = rss_info.get('rss_sites')
# 过滤订阅站点
if rss_sites and media_info.site not in rss_sites:
continue
# 有tmdbid时精确匹配
name = rss_info.get('name')
year = rss_info.get('year')
season = rss_info.get('season')
tmdbid = rss_info.get('tmdbid')
fuzzy_match = rss_info.get('fuzzy_match')
# 非模糊匹配
if not fuzzy_match:
if tmdbid and not tmdbid.startswith("DB:"):
if str(media_info.tmdb_id) != str(tmdbid):
continue
else:
# 匹配年份,年份可以为空
if year and str(year) != str(media_info.year):
continue
# 匹配名称
if name != media_info.title:
continue
# 匹配季,季可以为空
if season and season != media_info.get_season_string():
continue
# 模糊匹配
else:
# 匹配季,季可以为空
if season and season != "S00" and season != media_info.get_season_string():
continue
# 匹配年份
if year and str(year) != str(media_info.year):
continue
# 匹配关键字或正则表达式
search_title = f"{media_info.org_string} {media_info.title} {media_info.year}"
if not re.search(name, search_title, re.I) and name not in search_title:
continue
# 媒体匹配成功
match_flag = True
match_rss_info = rss_info
break
# 名称匹配成功,开始过滤
if match_flag:
# 解析种子详情
if site_parse:
# 检测Free
torrent_attr = self.sites.check_torrent_attr(torrent_url=media_info.page_url,
cookie=site_cookie,
ua=site_ua,
proxy=site_proxy)
if torrent_attr.get('2xfree'):
download_volume_factor = 0.0
upload_volume_factor = 2.0
elif torrent_attr.get('free'):
download_volume_factor = 0.0
upload_volume_factor = 1.0
else:
upload_volume_factor = 1.0
download_volume_factor = 1.0
if torrent_attr.get('hr'):
hit_and_run = True
# 设置属性
media_info.set_torrent_info(upload_volume_factor=upload_volume_factor,
download_volume_factor=download_volume_factor,
hit_and_run=hit_and_run)
# 订阅无过滤规则应用站点设置
filter_rule = match_rss_info.get('filter_rule') or site_filter_rule
filter_dict = {
"restype": match_rss_info.get('filter_restype'),
"pix": match_rss_info.get('filter_pix'),
"team": match_rss_info.get('filter_team'),
"rule": filter_rule
}
match_filter_flag, res_order, match_filter_msg = self.filter.check_torrent_filter(meta_info=media_info,
filter_args=filter_dict)
if not match_filter_flag:
match_msg.append(match_filter_msg)
return False, match_msg, match_rss_info
else:
match_msg.append("%s 识别为 %s %s 匹配订阅成功" % (
media_info.org_string,
media_info.get_title_string(),
media_info.get_season_episode_string()))
match_msg.append(f"种子描述:{media_info.subtitle}")
match_rss_info.update({
"res_order": res_order,
"filter_rule": filter_rule,
"upload_volume_factor": upload_volume_factor,
"download_volume_factor": download_volume_factor})
return True, match_msg, match_rss_info
else:
match_msg.append("%s 识别为 %s %s 不在订阅范围" % (
media_info.org_string,
media_info.get_title_string(),
media_info.get_season_episode_string()))
return False, match_msg, match_rss_info
def download_rss_torrent(self, rss_download_torrents, rss_no_exists):
"""
根据缺失情况以及匹配到的结果选择下载种子
"""
if not rss_download_torrents:
return
finished_rss_torrents = []
updated_rss_torrents = []
def __finish_rss(download_item):
"""
完成订阅
"""
if not download_item:
return
if not download_item.rssid \
or download_item.rssid in finished_rss_torrents:
return
finished_rss_torrents.append(download_item.rssid)
self.subscribe.finish_rss_subscribe(rssid=download_item.rssid,
media=download_item)
def __update_tv_rss(download_item, left_media):
"""
更新订阅集数
"""
if not download_item or not left_media:
return
if not download_item.rssid \
or download_item.rssid in updated_rss_torrents:
return
updated_rss_torrents.append(download_item.rssid)
self.subscribe.update_subscribe_tv_lack(rssid=download_item.rssid,
media_info=download_item,
seasoninfo=left_media)
def __update_over_edition(download_item):
"""
更新洗版订阅
"""
if not download_item:
return
if not download_item.rssid \
or download_item.rssid in updated_rss_torrents:
return
if download_item.get_episode_list():
return
updated_rss_torrents.append(download_item.rssid)
self.subscribe.update_subscribe_over_edition(rtype=download_item.type,
rssid=download_item.rssid,
media=download_item)
# 去重择优后开始添加下载
download_items, left_medias = self.downloader.batch_download(SearchType.RSS,
rss_download_torrents,
rss_no_exists)
# 批量删除订阅
if download_items:
for item in download_items:
if not item.rssid:
continue
if item.over_edition:
# 更新洗版订阅
__update_over_edition(item)
elif not left_medias or not left_medias.get(item.tmdb_id):
# 删除电视剧订阅
__finish_rss(item)
else:
# 更新电视剧缺失剧集
__update_tv_rss(item, left_medias.get(item.tmdb_id))
log.info("【Rss】实际下载了 %s 个资源" % len(download_items))
else:
log.info("【Rss】未下载到任何资源")