Commit 73b8debb authored by echel0n's avatar echel0n
Browse files

Added scene column back into TV show table

Bumped main database version to 13
Added scene option throughout SiCKRAGE
Added lock to queue worker, acquire and release lock when notifying workers
Fixed issue with removing episode from show episode cache
Removed QuickSearch cache class and database tables, using data to populate quicksearch entries from main database instead
parent 48db98bf
......@@ -20,7 +20,6 @@
# ##############################################################################
import asyncio
import datetime
import functools
import os
import platform
import re
......@@ -44,7 +43,6 @@ import sickrage
from sickrage.core.announcements import Announcements
from sickrage.core.api import API
from sickrage.core.auth import AuthServer
from sickrage.core.caches.quicksearch_cache import QuicksearchCache
from sickrage.core.common import SD, SKIPPED, WANTED
from sickrage.core.config import Config
from sickrage.core.databases.cache import CacheDB
......@@ -166,7 +164,6 @@ class Core(object):
self.auto_postprocessor = None
self.upnp_client = None
self.auth_server = None
self.quicksearch_cache = None
self.announcements = None
self.api = None
......@@ -208,7 +205,6 @@ class Core(object):
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
self.announcements = Announcements()
# authorization sso client
......@@ -295,9 +291,6 @@ class Core(object):
# set torrent client web url
torrent_webui_url(True)
# load quicksearch cache
self.quicksearch_cache.load()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
......@@ -552,9 +545,8 @@ class Core(object):
self.shows = {}
for query in session.query(MainDB.TVShow).with_entities(MainDB.TVShow.indexer_id, MainDB.TVShow.indexer, MainDB.TVShow.name):
try:
self.log.info('Loading show {} and building caches'.format(query.name))
self.log.info('Loading show {}'.format(query.name))
self.shows.update({(query.indexer_id, query.indexer): TVShow(query.indexer_id, query.indexer)})
self.quicksearch_cache.add_show(query.indexer_id)
except Exception as e:
self.log.debug('There was an error loading show: {}'.format(query.name))
......
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
import threading
from sqlalchemy import orm
import sickrage
from sickrage.core.databases.cache import CacheDB
from sickrage.core.media.util import showImage
from sickrage.core.tv.show.helpers import find_show, get_show_list
class QuicksearchCache(object):
def __init__(self):
self.name = "QUICKSEARCH-CACHE"
self.lock = threading.Lock()
self.cache = {
'shows': {},
'episodes': {}
}
def run(self):
# set thread name
threading.currentThread().setName(self.name)
self.load()
[self.add_show(show.indexer_id) for show in get_show_list()]
def load(self):
session = sickrage.app.cache_db.session()
for x in session.query(CacheDB.QuickSearchShow):
self.cache['shows'][x.showid] = x.as_dict()
for x in session.query(CacheDB.QuickSearchEpisode):
self.cache['episodes'][x.episodeid] = x.as_dict()
sickrage.app.log.debug("Loaded {} shows to QuickSearch cache".format(len(self.cache['shows'])))
sickrage.app.log.debug("Loaded {} episodes to QuickSearch cache".format(len(self.cache['episodes'])))
def get_shows(self, term):
return [d for d in self.cache['shows'].values() if d['name'] is not None and term.lower() in d['name'].lower()]
def get_episodes(self, term):
return [d for d in self.cache['episodes'].values() if d['name'] is not None and term.lower() in d['name'].lower()]
def add_show(self, indexer_id, update=False):
session = sickrage.app.cache_db.session()
show = find_show(indexer_id)
if indexer_id not in self.cache['shows']:
sickrage.app.log.debug("Adding show {} to QuickSearch cache".format(show.name))
elif update:
sickrage.app.log.debug("Updating show {} in QuickSearch cache".format(show.name))
else:
return
qsData = {
'category': 'shows',
'showid': indexer_id,
'seasons': len(set([s.season for s in show.episodes])),
'name': show.name,
'img': sickrage.app.config.web_root + showImage(indexer_id, 'poster_thumb').url
}
try:
dbData = session.query(CacheDB.QuickSearchShow).filter_by(showid=indexer_id).one()
dbData.update(**qsData)
except orm.exc.NoResultFound:
session.add(CacheDB.QuickSearchShow(**qsData))
finally:
session.commit()
self.cache['shows'][indexer_id] = qsData
sql_insert = []
sql_update = []
for e in show.episodes:
qsData = {
'category': 'episodes',
'showid': e.showid,
'episodeid': e.indexer_id,
'season': e.season,
'episode': e.episode,
'name': e.name,
'showname': show.name,
'img': sickrage.app.config.web_root + showImage(e.showid, 'poster_thumb').url
}
if e.indexer_id not in self.cache['episodes']:
sql_insert.append(qsData)
else:
sql_update.append(qsData)
self.cache['episodes'][e.indexer_id] = qsData
if len(sql_insert):
session.bulk_insert_mappings(CacheDB.QuickSearchEpisode, sql_insert)
session.commit()
if len(sql_update):
session.bulk_update_mappings(CacheDB.QuickSearchEpisode, sql_update)
session.commit()
def del_show(self, indexer_id):
session = sickrage.app.cache_db.session()
show = find_show(indexer_id)
sickrage.app.log.debug("Deleting show {} from QuickSearch cache".format(show.name))
# remove from database
session.query(CacheDB.QuickSearchShow).filter_by(showid=indexer_id).delete()
session.commit()
session.query(CacheDB.QuickSearchEpisode).filter_by(showid=indexer_id).delete()
session.commit()
if indexer_id in self.cache['shows'].copy():
del self.cache['shows'][indexer_id]
for k, v in self.cache['episodes'].copy().items():
if v['showid'] == indexer_id:
del self.cache['episodes'][k]
......@@ -80,7 +80,6 @@ class SearchFormats(object):
AIR_BY_DATE = 2
ANIME = 3
SPORTS = 4
SCENE = 5
COLLECTION = 6
search_format_strings = {
......@@ -88,7 +87,6 @@ class SearchFormats(object):
AIR_BY_DATE: 'Air By Date (Show.2010.03.02)',
ANIME: 'Anime (Show.265)',
SPORTS: 'Sports (Show.2010.03.02)',
SCENE: 'Scene Numbering (Show.S01E01)',
COLLECTION: 'Collection (Show.Series.1.1of10) or (Show.Series.1.Part.1)'
}
......
......@@ -123,6 +123,7 @@ class Config(object):
self.indexer_timeout = 120
self.search_format_default = 0
self.anime_default = False
self.scene_default = False
self.skip_downloaded_default = False
self.add_show_year_default = False
self.naming_multi_ep = False
......@@ -759,6 +760,7 @@ class Config(object):
'auto_update': True,
'tv_download_dir': '',
'naming_custom_abd': False,
'scene_default': False,
'skip_downloaded_default': False,
'add_show_year_default': False,
'naming_sports_pattern': '%SN - %A-D - %EN',
......@@ -1496,6 +1498,7 @@ class Config(object):
self.indexer_timeout = self.check_setting_int('General', 'indexer_timeout')
self.anime_default = self.check_setting_bool('General', 'anime_default')
self.search_format_default = self.check_setting_int('General', 'search_format_default')
self.scene_default = self.check_setting_bool('General', 'scene_default')
self.skip_downloaded_default = self.check_setting_bool('General', 'skip_downloaded_default')
self.add_show_year_default = self.check_setting_bool('General', 'add_show_year_default')
self.naming_pattern = self.check_setting_str('General', 'naming_pattern')
......@@ -2018,6 +2021,7 @@ class Config(object):
'indexer_timeout': int(self.indexer_timeout),
'anime_default': int(self.anime_default),
'search_format_default': int(self.search_format_default),
'scene_default': int(self.scene_default),
'skip_downloaded_default': int(self.skip_downloaded_default),
'add_show_year_default': int(self.add_show_year_default),
'enable_upnp': int(self.enable_upnp),
......
......@@ -29,7 +29,7 @@ class CacheDBBase(SRDatabaseBase):
class CacheDB(SRDatabase):
def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
super(CacheDB, self).__init__('cache', 7, db_type, db_prefix, db_host, db_port, db_username, db_password)
super(CacheDB, self).__init__('cache', 8, db_type, db_prefix, db_host, db_port, db_username, db_password)
CacheDBBase.metadata.create_all(self.engine)
for model in CacheDBBase._decl_class_registry.values():
if hasattr(model, '__tablename__'):
......@@ -106,27 +106,6 @@ class CacheDB(SRDatabase):
leechers = Column(Integer)
size = Column(Integer)
class QuickSearchShow(CacheDBBase):
__tablename__ = 'quicksearch_shows'
category = Column(Text)
showid = Column(Integer, index=True, primary_key=True)
seasons = Column(Integer)
name = Column(Text)
img = Column(Text)
class QuickSearchEpisode(CacheDBBase):
__tablename__ = 'quicksearch_episodes'
category = Column(Text)
showid = Column(Integer, index=True, primary_key=True)
episodeid = Column(Integer)
season = Column(Integer, index=True, primary_key=True)
episode = Column(Integer, index=True, primary_key=True)
name = Column(Text)
showname = Column(Text)
img = Column(Text)
class OAuth2Token(CacheDBBase):
__tablename__ = 'oauth2_token'
......
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from sqlalchemy import *
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
quicksearch_shows = Table('quicksearch_shows', meta, autoload=True)
if quicksearch_shows is not None:
quicksearch_shows.drop()
quicksearch_episodes = Table('quicksearch_episodes', meta, autoload=True)
if quicksearch_episodes is not None:
quicksearch_episodes.drop()
def downgrade(migrate_engine):
pass
......@@ -33,7 +33,7 @@ class MainDBBase(SRDatabaseBase):
class MainDB(SRDatabase):
def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
super(MainDB, self).__init__('main', 12, db_type, db_prefix, db_host, db_port, db_username, db_password)
super(MainDB, self).__init__('main', 13, db_type, db_prefix, db_host, db_port, db_username, db_password)
MainDBBase.metadata.create_all(self.engine)
for model in MainDBBase._decl_class_registry.values():
if hasattr(model, '__tablename__'):
......@@ -86,6 +86,7 @@ class MainDB(SRDatabase):
flatten_folders = Column(Boolean, default=0)
paused = Column(Boolean, default=0)
search_format = Column(Integer, default=SearchFormats.STANDARD)
scene = Column(Boolean, default=0)
anime = Column(Boolean, default=0)
subtitles = Column(Boolean, default=0)
dvdorder = Column(Boolean, default=0)
......
......@@ -24,13 +24,13 @@ def upgrade(migrate_engine):
if row.anime == 1 and not row.scene == 1:
value = SearchFormats.ANIME
elif row.anime == 1 and row.scene == 1:
value = SearchFormats.SCENE
value = 5
elif row.sports == 1:
value = SearchFormats.SPORTS
elif row.air_by_date == 1:
value = SearchFormats.AIR_BY_DATE
elif row.scene == 1:
value = SearchFormats.SCENE
value = 5
else:
value = SearchFormats.STANDARD
......
from sqlalchemy import *
from sickrage.core.common import SearchFormats
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
tv_shows = Table('tv_shows', meta, autoload=True)
if not hasattr(tv_shows.c, 'scene'):
scene = Column('scene', Boolean, default=0)
scene.create(tv_shows)
with migrate_engine.begin() as conn:
for row in migrate_engine.execute(tv_shows.select()):
if row.search_format == 5:
conn.execute(tv_shows.update().where(tv_shows.c.indexer_id == row.indexer_id).values(scene=1, search_format=SearchFormats.STANDARD))
def downgrade(migrate_engine):
pass
......@@ -31,7 +31,6 @@ from sqlalchemy import orm
import sickrage
from sickrage.core import common
from sickrage.core.common import SearchFormats
from sickrage.core.databases.main import MainDB
from sickrage.core.helpers import remove_extension, strip_accents
from sickrage.core.nameparser import regexes
......@@ -278,7 +277,7 @@ class NameParser(object):
s = season_number
e = epNo
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
if show_obj.scene and not skip_scene_detection:
(s, e) = get_indexer_numbering(show_obj.indexer_id,
show_obj.indexer,
season_number,
......@@ -290,7 +289,7 @@ class NameParser(object):
for epAbsNo in best_result.ab_episode_numbers:
a = epAbsNo
if show_obj.search_format == SearchFormats.SCENE:
if show_obj.scene:
scene_result = show_obj.get_scene_exception_by_name(best_result.series_name)
if scene_result:
a = get_indexer_absolute_numbering(show_obj.indexer_id,
......@@ -308,7 +307,7 @@ class NameParser(object):
s = best_result.season_number
e = epNo
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
if show_obj.scene and not skip_scene_detection:
(s, e) = get_indexer_numbering(show_obj.indexer_id,
show_obj.indexer,
best_result.season_number,
......@@ -347,7 +346,7 @@ class NameParser(object):
best_result.episode_numbers = new_episode_numbers
best_result.season_number = new_season_numbers[0]
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
if show_obj.scene and not skip_scene_detection:
sickrage.app.log.debug("Scene converted parsed result {} into {}".format(best_result.original_name, best_result))
# CPU sleep
......
......@@ -339,7 +339,6 @@ class Worker(object):
finally:
sickrage.app.log.debug("Worker " + str(self.id) + " task completed...")
self.status = WorkerStatus.IDLE
self.task = None
self.notify()
......
......@@ -29,7 +29,7 @@ from enum import Enum
import sickrage
from sickrage.core.common import WANTED
from sickrage.core.exceptions import CantRefreshShowException, CantRemoveShowException, CantUpdateShowException, EpisodeDeletedException, \
MultipleShowObjectsException
MultipleShowObjectsException, EpisodeNotFoundException
from sickrage.core.queues import Queue, Task, TaskPriority
from sickrage.core.scene_numbering import xem_refresh
from sickrage.core.traktapi import TraktAPI
......@@ -118,7 +118,7 @@ class ShowQueue(Queue):
def add_show(self, indexer, indexer_id, showDir, default_status=None, quality=None, flatten_folders=None, lang=None, subtitles=None,
sub_use_sr_metadata=None, anime=None, search_format=None, dvdorder=None, paused=None, blacklist=None, whitelist=None,
default_status_after=None, skip_downloaded=None):
default_status_after=None, scene=None, skip_downloaded=None):
if lang is None:
lang = sickrage.app.config.indexer_default_language
......@@ -139,6 +139,7 @@ class ShowQueue(Queue):
blacklist=blacklist,
whitelist=whitelist,
default_status_after=default_status_after,
scene=scene,
skip_downloaded=skip_downloaded))
def remove_show(self, indexer_id, full=False):
......@@ -198,7 +199,7 @@ class ShowTask(Task):
class ShowTaskAdd(ShowTask):
def __init__(self, indexer, indexer_id, showDir, default_status, quality, flatten_folders, lang, subtitles, sub_use_sr_metadata, anime,
dvdorder, search_format, paused, blacklist, whitelist, default_status_after, skip_downloaded):
dvdorder, search_format, paused, blacklist, whitelist, default_status_after, scene, skip_downloaded):
super(ShowTaskAdd, self).__init__(None, ShowTaskActions.ADD)
self.indexer = indexer
......@@ -217,6 +218,7 @@ class ShowTaskAdd(ShowTask):
self.blacklist = blacklist
self.whitelist = whitelist
self.default_status_after = default_status_after
self.scene = scene
self.skip_downloaded = skip_downloaded
self.priority = TaskPriority.HIGH
......@@ -311,6 +313,7 @@ class ShowTaskAdd(ShowTask):
show_obj.anime = self.anime or sickrage.app.config.anime_default
show_obj.dvdorder = self.dvdorder
show_obj.search_format = self.search_format or sickrage.app.config.search_format_default
show_obj.scene = self.scene or sickrage.app.config.scene_default
show_obj.skip_downloaded = self.skip_downloaded or sickrage.app.config.skip_downloaded_default
show_obj.paused = self.paused
......@@ -395,8 +398,6 @@ class ShowTaskAdd(ShowTask):
show_obj.save()
sickrage.app.quicksearch_cache.add_show(show_obj.indexer_id)
sickrage.app.log.info("Finished adding show {} in {}s from show dir: {}".format(self.show_name, round(time.time() - start_time, 2), self.showDir))
def _finish_early(self):
......@@ -541,15 +542,15 @@ class ShowTaskUpdate(ShowTask):
for curSeason in db_episodes:
for curEpisode in db_episodes[curSeason]:
sickrage.app.log.info("Permanently deleting episode " + str(curSeason) + "x" + str(curEpisode) + " from the database")
try:
show_obj.get_episode(curSeason, curEpisode).delete_episode()
except EpisodeDeletedException:
continue
episode_obj = show_obj.get_episode(curSeason, curEpisode, no_create=True)
if episode_obj:
try:
episode_obj.delete_episode()
except EpisodeDeletedException:
continue
show_obj.retrieve_scene_exceptions()
sickrage.app.quicksearch_cache.add_show(show_obj.indexer_id, update=True)
sickrage.app.log.info("Finished updates in {}s for show: {}".format(round(time.time() - start_time, 2), show_obj.name))
......@@ -580,8 +581,6 @@ class ShowTaskForceRemove(ShowTask):
sickrage.app.log.info("Removing show: {}".format(show_obj.name))
sickrage.app.quicksearch_cache.del_show(show_obj.indexer_id)
show_obj.delete_show(full=self.full)
if sickrage.app.config.use_trakt:
......
......@@ -25,7 +25,6 @@ import traceback
from sqlalchemy import orm
import sickrage
from sickrage.core.common import SearchFormats
from sickrage.core.databases.main import MainDB
from sickrage.core.exceptions import SiCKRAGETVEpisodeException
from sickrage.core.tv.show.helpers import find_show
......@@ -50,7 +49,7 @@ def get_scene_numbering(indexer_id, indexer, season, episode, fallback_to_xem=Tr
return season, episode
show_obj = find_show(int(indexer_id))
if show_obj and not show_obj.search_format == SearchFormats.SCENE:
if show_obj and not show_obj.scene:
return season, episode
result = find_scene_numbering(int(indexer_id), int(indexer), season, episode)
......@@ -103,7 +102,7 @@ def get_scene_absolute_numbering(indexer_id, indexer, absolute_number, fallback_
indexer = int(indexer)
show_obj = find_show(indexer_id)
if show_obj and not show_obj.search_format == SearchFormats.SCENE:
if show_obj and not show_obj.scene:
return absolute_number
result = find_scene_absolute_numbering(indexer_id, indexer, absolute_number)
......
......@@ -434,6 +434,7 @@ class TraktSearcher(object):
flatten_folders=int(sickrage.app.config.flatten_folders_default),
paused=sickrage.app.config.trakt_start_paused,
default_status_after=status,
scene=sickrage.app.config.scene_default,
skip_downloaded=sickrage.app.config.skip_downloaded_default)
else:
sickrage.app.log.warning(
......
......@@ -32,15 +32,11 @@ from sqlalchemy import orm
import sickrage
from sickrage.core.common import NAMING_EXTEND, NAMING_LIMITED_EXTEND, NAMING_LIMITED_EXTEND_E_PREFIXED, NAMING_DUPLICATE, NAMING_SEPARATED_REPEAT, \
SearchFormats
from sickrage.core.common import Quality, SKIPPED, UNKNOWN, UNAIRED, statusStrings
SearchFormats, Quality, SKIPPED, UNKNOWN, UNAIRED, statusStrings
from sickrage.core.databases.main import MainDB
from sickrage.core.exceptions import EpisodeNotFoundException, EpisodeDeletedException
from sickrage.core.exceptions import NoNFOException
from sickrage.core.helpers import file_size
from sickrage.core.helpers import is_media_file, try_int, safe_getattr
from sickrage.core.exceptions import EpisodeNotFoundException, EpisodeDeletedException, NoNFOException
from sickrage.core.helpers import replace_extension, modify_file_timestamp, sanitize_scene_name, remove_non_release_groups, \
remove_extension, sanitize_file_name, make_dirs, move_file, delete_empty_folders
remove_extension, sanitize_file_name, make_dirs, move_file, delete_empty_folders, file_size, is_media_file, try_int, safe_getattr
from sickrage.core.tv.show.helpers import find_show
from sickrage.indexers import IndexerApi
from sickrage.indexers.exceptions import indexer_seasonnotfound, indexer_episodenotfound
......@@ -668,11 +664,9 @@ class TVEpisode(object):
sickrage.app.log.warning('Unable to delete episode file %s: %s / %s' % (self.location, repr(e), str(e)))
# delete myself from show episode cache
try:
if self.indexer_id in self.show.episodes:
sickrage.app.log.debug("Deleting %s S%02dE%02d from the shows episode cache" % (self.show.name, self.season or 0, self.episode or 0))
del self.show.episodes[self.show.episodes.index(self)]
except (IndexError, ValueError) as e:
pass
del self.show._episodes[self.indexer_id]
# delete myself from the database
sickrage.app.log.debug("Deleting %s S%02dE%02d from the DB" % (self.show.name, self.season or 0, self.episode or 0))
......
......@@ -43,7 +43,6 @@ from sickrage.core.common import Quality, SKIPPED, WANTED, UNKNOWN, DOWNLOADED,
from sickrage.core.databases.main import MainDB
from sickrage.core.exceptions import ShowNotFoundException, EpisodeNotFoundException, EpisodeDeletedException, MultipleEpisodesInDatabaseException
from sickrage.core.helpers import list_media_files, is_media_file, try_int, safe_getattr
from sickrage.core.nameparser import NameParser, InvalidNameException, InvalidShowException
from sickrage.core.tv.episode import TVEpisode
from sickrage.indexers import IndexerApi
from sickrage.indexers.exceptions import indexer_attributenotfound, indexer_exception
......@@ -187,6 +186,14 @@ class TVShow(object):
def paused(self, value):