288 lines
14 KiB
Python
288 lines
14 KiB
Python
import datetime
|
||
import random
|
||
from threading import Lock
|
||
from time import sleep
|
||
|
||
import log
|
||
from app.downloader import Downloader
|
||
from app.helper import DbHelper
|
||
from app.media import Media, DouBan
|
||
from app.media.meta import MetaInfo
|
||
from app.message import Message
|
||
from app.searcher import Searcher
|
||
from app.subscribe import Subscribe
|
||
from app.utils import ExceptionUtils
|
||
from app.utils.types import SearchType, MediaType
|
||
from config import Config
|
||
|
||
lock = Lock()
|
||
|
||
|
||
class DoubanSync:
|
||
douban = None
|
||
searcher = None
|
||
media = None
|
||
downloader = None
|
||
dbhelper = None
|
||
subscribe = None
|
||
message = None
|
||
_interval = None
|
||
_auto_search = None
|
||
_auto_rss = None
|
||
_users = None
|
||
_days = None
|
||
_types = None
|
||
|
||
def __init__(self):
|
||
self.init_config()
|
||
|
||
def init_config(self):
|
||
self.douban = DouBan()
|
||
self.searcher = Searcher()
|
||
self.downloader = Downloader()
|
||
self.media = Media()
|
||
self.message = Message()
|
||
self.dbhelper = DbHelper()
|
||
self.subscribe = Subscribe()
|
||
douban = Config().get_config('douban')
|
||
if douban:
|
||
# 同步间隔
|
||
self._interval = int(douban.get('interval')) if str(douban.get('interval')).isdigit() else None
|
||
self._auto_search = douban.get('auto_search')
|
||
self._auto_rss = douban.get('auto_rss')
|
||
# 用户列表
|
||
users = douban.get('users')
|
||
if users:
|
||
if not isinstance(users, list):
|
||
users = [users]
|
||
self._users = users
|
||
# 时间范围
|
||
self._days = int(douban.get('days')) if str(douban.get('days')).isdigit() else None
|
||
# 类型
|
||
types = douban.get('types')
|
||
if types:
|
||
self._types = types.split(',')
|
||
|
||
def sync(self):
|
||
"""
|
||
同步豆瓣数据
|
||
"""
|
||
if not self._interval:
|
||
log.info("【Douban】豆瓣配置：同步间隔未配置或配置不正确")
|
||
return
|
||
with lock:
|
||
log.info("【Douban】开始同步豆瓣数据...")
|
||
# 拉取豆瓣数据
|
||
medias = self.__get_all_douban_movies()
|
||
# 开始检索
|
||
for media in medias:
|
||
if not media or not media.get_name():
|
||
continue
|
||
try:
|
||
# 查询数据库状态，已经加入RSS的不处理
|
||
search_state = self.dbhelper.get_douban_search_state(media.get_name(), media.year)
|
||
if not search_state or search_state[0] == "NEW":
|
||
if self._auto_search:
|
||
# 需要检索
|
||
if media.begin_season:
|
||
subtitle = "第%s季" % media.begin_season
|
||
else:
|
||
subtitle = None
|
||
media_info = self.media.get_media_info(title="%s %s" % (media.get_name(), media.year or ""),
|
||
subtitle=subtitle,
|
||
mtype=media.type)
|
||
# 不需要自动加订阅，则直接搜索
|
||
if not media_info or not media_info.tmdb_info:
|
||
log.warn("【Douban】%s 未查询到媒体信息" % media.get_name())
|
||
continue
|
||
# 检查是否存在，电视剧返回不存在的集清单
|
||
exist_flag, no_exists, _ = self.downloader.check_exists_medias(meta_info=media_info)
|
||
# 已经存在
|
||
if exist_flag:
|
||
# 更新为已下载状态
|
||
log.info("【Douban】%s 已存在" % media.get_name())
|
||
self.dbhelper.insert_douban_media_state(media, "DOWNLOADED")
|
||
continue
|
||
if not self._auto_rss:
|
||
# 合并季
|
||
media_info.begin_season = media.begin_season
|
||
# 开始检索
|
||
search_result, no_exists, search_count, download_count = self.searcher.search_one_media(
|
||
media_info=media_info,
|
||
in_from=SearchType.DB,
|
||
no_exists=no_exists,
|
||
user_name=media_info.user_name)
|
||
if search_result:
|
||
# 下载全了更新为已下载，没下载全的下次同步再次搜索
|
||
self.dbhelper.insert_douban_media_state(media, "DOWNLOADED")
|
||
else:
|
||
# 需要加订阅，则由订阅去检索
|
||
log.info(
|
||
"【Douban】%s %s 更新到%s订阅中..." % (media.get_name(), media.year, media.type.value))
|
||
code, msg, _ = self.subscribe.add_rss_subscribe(mtype=media.type,
|
||
name=media.get_name(),
|
||
year=media.year,
|
||
season=media.begin_season,
|
||
mediaid=f"DB:{media.douban_id}")
|
||
if code != 0:
|
||
log.error("【Douban】%s 添加订阅失败：%s" % (media.get_name(), msg))
|
||
# 订阅已存在
|
||
if code == 9:
|
||
self.dbhelper.insert_douban_media_state(media, "RSS")
|
||
else:
|
||
# 发送订阅消息
|
||
self.message.send_rss_success_message(in_from=SearchType.DB,
|
||
media_info=media)
|
||
# 插入为已RSS状态
|
||
self.dbhelper.insert_douban_media_state(media, "RSS")
|
||
else:
|
||
# 不需要检索
|
||
if self._auto_rss:
|
||
# 加入订阅，使状态为R
|
||
log.info("【Douban】%s %s 更新到%s订阅中..." % (
|
||
media.get_name(), media.year, media.type.value))
|
||
code, msg, _ = self.subscribe.add_rss_subscribe(mtype=media.type,
|
||
name=media.get_name(),
|
||
year=media.year,
|
||
season=media.begin_season,
|
||
mediaid=f"DB:{media.douban_id}",
|
||
state="R")
|
||
if code != 0:
|
||
log.error("【Douban】%s 添加订阅失败：%s" % (media.get_name(), msg))
|
||
# 订阅已存在
|
||
if code == 9:
|
||
self.dbhelper.insert_douban_media_state(media, "RSS")
|
||
else:
|
||
# 发送订阅消息
|
||
self.message.send_rss_success_message(in_from=SearchType.DB,
|
||
media_info=media)
|
||
# 插入为已RSS状态
|
||
self.dbhelper.insert_douban_media_state(media, "RSS")
|
||
elif not search_state:
|
||
log.info("【Douban】%s %s 更新到%s列表中..." % (
|
||
media.get_name(), media.year, media.type.value))
|
||
self.dbhelper.insert_douban_media_state(media, "NEW")
|
||
|
||
else:
|
||
log.info("【Douban】%s %s 已处理过" % (media.get_name(), media.year))
|
||
except Exception as err:
|
||
log.error("【Douban】%s %s 处理失败：%s" % (media.get_name(), media.year, str(err)))
|
||
continue
|
||
log.info("【Douban】豆瓣数据同步完成")
|
||
|
||
def __get_all_douban_movies(self):
|
||
"""
|
||
获取每一个用户的每一个类型的豆瓣标记
|
||
:return: 检索到的媒体信息列表（不含TMDB信息）
|
||
"""
|
||
if not self._interval \
|
||
or not self._days \
|
||
or not self._users \
|
||
or not self._types:
|
||
log.warn("【Douban】豆瓣未配置或配置不正确")
|
||
return []
|
||
# 返回媒体列表
|
||
media_list = []
|
||
# 豆瓣ID列表
|
||
douban_ids = {}
|
||
# 每页条数
|
||
perpage_number = 15
|
||
# 每一个用户
|
||
for user in self._users:
|
||
if not user:
|
||
continue
|
||
# 查询用户名称
|
||
user_name = ""
|
||
userinfo = self.douban.get_user_info(userid=user)
|
||
if userinfo:
|
||
user_name = userinfo.get("name")
|
||
# 每一个类型成功数量
|
||
user_succnum = 0
|
||
for mtype in self._types:
|
||
if not mtype:
|
||
continue
|
||
log.info(f"【Douban】开始获取 {user_name or user} 的 {mtype} 数据...")
|
||
# 开始序号
|
||
start_number = 0
|
||
# 类型成功数量
|
||
user_type_succnum = 0
|
||
# 每一页
|
||
while True:
|
||
# 页数
|
||
page_number = int(start_number / perpage_number + 1)
|
||
# 当前页成功数量
|
||
sucess_urlnum = 0
|
||
# 是否继续下一页
|
||
continue_next_page = True
|
||
log.debug(f"【Douban】开始解析第 {page_number} 页数据...")
|
||
try:
|
||
items = self.douban.get_douban_wish(dtype=mtype, userid=user, start=start_number, wait=True)
|
||
if not items:
|
||
log.warn(f"【Douban】第 {page_number} 页未获取到数据")
|
||
break
|
||
# 解析豆瓣ID
|
||
for item in items:
|
||
# 时间范围
|
||
date = item.get("date")
|
||
if not date:
|
||
continue_next_page = False
|
||
break
|
||
else:
|
||
mark_date = datetime.datetime.strptime(date, '%Y-%m-%d')
|
||
if not (datetime.datetime.now() - mark_date).days < int(self._days):
|
||
continue_next_page = False
|
||
break
|
||
doubanid = item.get("id")
|
||
if str(doubanid).isdigit():
|
||
log.info("【Douban】解析到媒体：%s" % doubanid)
|
||
if doubanid not in douban_ids:
|
||
douban_ids[doubanid] = {
|
||
"user_name": user_name
|
||
}
|
||
sucess_urlnum += 1
|
||
user_type_succnum += 1
|
||
user_succnum += 1
|
||
log.debug(
|
||
f"【Douban】{user_name or user} 第 {page_number} 页解析完成，共获取到 {sucess_urlnum} 个媒体")
|
||
except Exception as err:
|
||
ExceptionUtils.exception_traceback(err)
|
||
log.error(f"【Douban】{user_name or user} 第 {page_number} 页解析出错：%s" % str(err))
|
||
break
|
||
# 继续下一页
|
||
if continue_next_page:
|
||
start_number += perpage_number
|
||
else:
|
||
break
|
||
# 当前类型解析结束
|
||
log.debug(f"【Douban】用户 {user_name or user} 的 {mtype} 解析完成，共获取到 {user_type_succnum} 个媒体")
|
||
log.info(f"【Douban】用户 {user_name or user} 解析完成，共获取到 {user_succnum} 个媒体")
|
||
log.info(f"【Douban】所有用户解析完成，共获取到 {len(douban_ids)} 个媒体")
|
||
# 查询豆瓣详情
|
||
for doubanid, info in douban_ids.items():
|
||
douban_info = self.douban.get_douban_detail(doubanid=doubanid, wait=True)
|
||
# 组装媒体信息
|
||
if not douban_info:
|
||
log.warn("【Douban】%s 未正确获取豆瓣详细信息，尝试使用网页获取" % doubanid)
|
||
douban_info = self.douban.get_media_detail_from_web(doubanid)
|
||
if not douban_info:
|
||
log.warn("【Douban】%s 无权限访问，需要配置豆瓣Cookie" % doubanid)
|
||
# 随机休眠
|
||
sleep(round(random.uniform(1, 5), 1))
|
||
continue
|
||
media_type = MediaType.TV if douban_info.get("episodes_count") else MediaType.MOVIE
|
||
log.info("【Douban】%s：%s %s".strip() % (media_type.value, douban_info.get("title"), douban_info.get("year")))
|
||
meta_info = MetaInfo(title="%s %s" % (douban_info.get("title"), douban_info.get("year") or ""))
|
||
meta_info.douban_id = doubanid
|
||
meta_info.type = media_type
|
||
meta_info.overview = douban_info.get("intro")
|
||
meta_info.poster_path = douban_info.get("cover_url")
|
||
rating = douban_info.get("rating", {}) or {}
|
||
meta_info.vote_average = rating.get("value") or ""
|
||
meta_info.imdb_id = douban_info.get("imdbid")
|
||
meta_info.user_name = info.get("user_name")
|
||
if meta_info not in media_list:
|
||
media_list.append(meta_info)
|
||
# 随机休眠
|
||
sleep(round(random.uniform(1, 5), 1))
|
||
return media_list
|