Commit 0ab3a377 authored by echel0n's avatar echel0n
Browse files

Refactored exception handling for search providers

Refactored core web session exception handling
Feature added that allows searching episodes by collection format
Refactored name parser match scoring
Refactored name parser regex placement for mvgroup
Refactored show search types into SearchFormats class
Refactored main db to v12 and added in code to migrate old-style show search options to new-style show search formats
Refactored core API web session exception handling
Refactored providers by removing archetorrent and elitetorrent
Fixed season folder show option, needs to be invested as it controls flattening of folders show option
Refactored requirements.txt, updated misc packages
parent 5449bcc6
......@@ -5,7 +5,7 @@ CacheControl == 0.12.6
cloudscraper == 1.2.34
configobj == 5.0.6
feedparser == 6.0.0b3
guessit == 3.1.0
guessit == 3.1.1
hachoir == 3.1.1
Mako == 1.1.2
markdown2 == 2.3.8
......@@ -13,7 +13,7 @@ oauth2 == 1.9.0.post1
profilehooks == 1.11.2
Send2Trash == 1.5.0
six == 1.14.0
subliminal == 2.0.5
subliminal == 2.1.0
tornado == 6.0.4
xmltodict == 0.12.0
MultipartPostHandler == 0.1.0
......@@ -30,11 +30,11 @@ lockfile == 0.12.2
requests == 2.23.0
fake-useragent == 0.1.11
html5lib == 1.0.1
arrow == 0.15.5
arrow == 0.15.6
unidecode == 1.1.1
twilio == 6.38.1
twilio == 6.39.0
chardet == 3.0.4
pytz == 2019.3
pytz == 2020.1
tzlocal == 2.1b1
raven == 6.10.0
python-keycloak-client == 0.2.3
......
......@@ -24,7 +24,6 @@ import threading
from sqlalchemy import orm
import sickrage
from sickrage.core.api import APIError
from sickrage.core.databases.cache import CacheDB
......@@ -76,16 +75,13 @@ class Announcements(object):
def worker(self, force):
threading.currentThread().setName(self.name)
try:
resp = sickrage.app.api.announcement.get_announcements()
if resp and 'data' in resp:
for announcement in resp['data']:
if announcement['enabled']:
self.add(announcement['hash'], announcement['title'], announcement['description'], announcement['image'], announcement['date'])
else:
self.clear(announcement['hash'])
except APIError as e:
pass
resp = sickrage.app.api.announcement.get_announcements()
if resp and 'data' in resp:
for announcement in resp['data']:
if announcement['enabled']:
self.add(announcement['hash'], announcement['title'], announcement['description'], announcement['image'], announcement['date'])
else:
self.clear(announcement['hash'])
def add(self, ahash, title, description, image, date):
session = sickrage.app.cache_db.session()
......
import collections
import time
import traceback
from urllib.parse import urljoin
import certifi
import errno
import oauthlib.oauth2
import requests
import requests.exceptions
from keycloak.exceptions import KeycloakClientError
from oauthlib.oauth2 import MissingTokenError, InvalidClientIdError, TokenExpiredError, InvalidGrantError, OAuth2Token
from requests_oauthlib import OAuth2Session
from sqlalchemy import orm
......@@ -83,7 +85,7 @@ class API(object):
'refresh_token': value.get('refresh_token'),
'expires_in': value.get('expires_in'),
'expires_at': value.get('expires_at', int(time.time() + value.get('expires_in'))),
'scope': value.scope if isinstance(value, OAuth2Token) else value.get('scope')
'scope': value.scope if isinstance(value, oauthlib.oauth2.OAuth2Token) else value.get('scope')
}
session = sickrage.app.cache_db.session()
......@@ -147,19 +149,22 @@ class API(object):
return self.request('POST', 'account/private-key', data=dict({'privatekey': privatekey}))
def request(self, method, url, timeout=30, **kwargs):
latest_exception = None
if not self.token:
return
for i in range(3):
url = urljoin(self.api_base, "/".join([self.api_version, url]))
for i in range(5):
resp = None
try:
if not self.health:
latest_exception = "SiCKRAGE backend API is currently unreachable ..."
if i > 3:
sickrage.app.log.debug('SiCKRAGE backend API is currently unreachable')
return None
continue
resp = self.session.request(method, urljoin(self.api_base, "/".join([self.api_version, url])), timeout=timeout, verify=False,
hooks={'response': self.throttle_hook}, **kwargs)
resp = self.session.request(method, url, timeout=timeout, verify=False, hooks={'response': self.throttle_hook}, **kwargs)
resp.raise_for_status()
if resp.status_code == 204:
......@@ -169,40 +174,53 @@ class API(object):
return resp.json()
except ValueError:
return resp.content
except TokenExpiredError:
except oauthlib.oauth2.TokenExpiredError:
self.refresh_token()
except (InvalidClientIdError, MissingTokenError, InvalidGrantError) as e:
latest_exception = "Invalid token error, please re-authenticate by logging out then logging back in from web-ui"
break
except requests.exceptions.ReadTimeout:
time.sleep(1)
except (oauthlib.oauth2.InvalidClientIdError, oauthlib.oauth2.MissingTokenError, oauthlib.oauth2.InvalidGrantError) as e:
sickrage.app.log.warning("Invalid token error, please re-authenticate by logging out then logging back in from web-ui")
return resp or e.response
except requests.exceptions.ReadTimeout as e:
if i > 3:
sickrage.app.log.debug('Error connecting to url {url} Error: {err_msg}'.format(url=url, err_msg=e))
return resp or e.response
timeout += timeout
time.sleep(1)
except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
error_message = e.response.text
if status_code == 403 and "login-pf-page" in error_message:
self.refresh_token()
time.sleep(1)
continue
if 'application/json' in e.response.headers.get('content-type', ''):
json_data = e.response.json().get('error', {})
status_code = json_data.get('status', status_code)
error_message = json_data.get('message', error_message)
latest_exception = APIError(status=status_code, message=error_message)
e = APIError(status=status_code, message=error_message)
if 400 <= status_code < 500:
break
except requests.exceptions.RequestException as e:
latest_exception = e
except ConnectionError:
pass
time.sleep(1)
sickrage.app.log.debug('The response returned a non-200 response while requesting url {url} Error: {err_msg!r}'.format(url=url, err_msg=e))
return resp or e.response
except requests.exceptions.ConnectionError as e:
if i > 3:
sickrage.app.log.debug('Error connecting to url {url} Error: {err_msg}'.format(url=url, err_msg=e))
return resp or e.response
if latest_exception:
if isinstance(latest_exception, APIError):
raise latest_exception
sickrage.app.log.warning('{!r}'.format(latest_exception))
time.sleep(1)
except requests.exceptions.RequestException as e:
sickrage.app.log.debug('Error requesting url {url} Error: {err_msg}'.format(url=url, err_msg=e))
return resp or e.response
except Exception as e:
if (isinstance(e, collections.Iterable) and 'ECONNRESET' in e) or (getattr(e, 'errno', None) == errno.ECONNRESET):
sickrage.app.log.warning('Connection reset by peer accessing url {url} Error: {err_msg}'.format(url=url, err_msg=e))
else:
sickrage.app.log.info('Unknown exception in url {url} Error: {err_msg}'.format(url=url, err_msg=e))
sickrage.app.log.debug(traceback.format_exc())
return None
@staticmethod
def throttle_hook(response, **kwargs):
......
......@@ -18,6 +18,9 @@ class APIError(Exception):
def __unicode__(self):
return self.__class__.__name__ + ': ' + self.message
def __repr__(self):
return self.__unicode__()
class APIResourceDoesNotExist(APIError):
"""Custom exception when resource is not found."""
......
......@@ -268,10 +268,9 @@ class TVCache(object):
# get data from external database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
dbData += sickrage.app.api.provider_cache.get(self.providerID, show_id, season, episode)['data']
except Exception:
pass
resp = sickrage.app.api.provider_cache.get(self.providerID, show_id, season, episode)
if resp and 'data' in resp:
dbData += resp['data']
# get data from internal database
session = sickrage.app.cache_db.session()
......
......@@ -19,15 +19,14 @@
import operator
import pathlib
import re
from collections import UserDict
from functools import reduce
import pathlib
from sickrage.core.helpers.metadata import get_file_metadata, get_resolution
### CPU Presets for sleep timers
# CPU Presets for sleep timers
cpu_presets = {
'HIGH': 0.05,
'NORMAL': 0.02,
......@@ -42,11 +41,11 @@ dateFormat = '%Y-%m-%d'
dateTimeFormat = '%Y-%m-%d %H:%M:%S'
timeFormat = '%A %I:%M %p'
### Other constants
# Other constants
MULTI_EP_RESULT = -1
SEASON_RESULT = -2
### Episode statuses
# Episode statuses
UNKNOWN = -1 # should never happen
UNAIRED = 1 # episodes that haven't aired yet
SNATCHED = 2 # qualified with quality
......@@ -76,7 +75,24 @@ multiEpStrings = {NAMING_REPEAT: _("Repeat"),
NAMING_LIMITED_EXTEND_E_PREFIXED: _("Extend (Limited, E-prefixed)")}
# pylint: disable=W0232
class SearchFormats(object):
STANDARD = 1
AIR_BY_DATE = 2
ANIME = 3
SPORTS = 4
SCENE = 5
COLLECTION = 6
search_format_strings = {
STANDARD: 'Standard (Show.S01E01)',
AIR_BY_DATE: 'Air By Date (Show.2010.03.02)',
ANIME: 'Anime (Show.265)',
SPORTS: 'Sports (Show.03.02.2010)',
SCENE: 'Scene Numbering (Show.S01E01)',
COLLECTION: 'Collection (Show.Series.1.1of10)'
}
class Quality(object):
NONE = 0 # 0
SDTV = 1 # 1
......
......@@ -37,7 +37,7 @@ import rarfile
from configobj import ConfigObj
import sickrage
from sickrage.core.common import SD, WANTED, SKIPPED, Quality
from sickrage.core.common import SD, WANTED, SKIPPED, Quality, SearchFormats
from sickrage.core.helpers import make_dir, generate_secret, auto_type, get_lan_ip, extract_zipfile, try_int, checkbox_to_value, generate_api_key, \
backup_versioned_file, encryption, move_file
from sickrage.core.websession import WebSession
......@@ -113,7 +113,7 @@ class Config(object):
self.subtitles_default = False
self.indexer_default = 0
self.indexer_timeout = 120
self.scene_default = False
self.search_format_default = 0
self.anime_default = False
self.skip_downloaded_default = False
self.add_show_year_default = False
......@@ -823,7 +823,7 @@ class Config(object):
'log_nr': 5,
'git_newver': False,
'git_reset': True,
'scene_default': False,
'search_format_default': SearchFormats.STANDARD,
'skip_removed_files': False,
'status_default_after': WANTED,
'last_db_compact': 0,
......@@ -1459,7 +1459,7 @@ class Config(object):
self.indexer_default = self.check_setting_int('General', 'indexer_default')
self.indexer_timeout = self.check_setting_int('General', 'indexer_timeout')
self.anime_default = self.check_setting_bool('General', 'anime_default')
self.scene_default = self.check_setting_bool('General', 'scene_default')
self.search_format_default = self.check_setting_int('General', 'search_format_default')
self.skip_downloaded_default = self.check_setting_bool('General', 'skip_downloaded_default')
self.add_show_year_default = self.check_setting_bool('General', 'add_show_year_default')
self.naming_pattern = self.check_setting_str('General', 'naming_pattern')
......@@ -1967,7 +1967,7 @@ class Config(object):
'indexer_default': int(self.indexer_default),
'indexer_timeout': int(self.indexer_timeout),
'anime_default': int(self.anime_default),
'scene_default': int(self.scene_default),
'search_format_default': int(self.search_format_default),
'skip_downloaded_default': int(self.skip_downloaded_default),
'add_show_year_default': int(self.add_show_year_default),
'enable_upnp': int(self.enable_upnp),
......
......@@ -22,6 +22,7 @@ from sqlalchemy.ext.declarative import as_declarative
from sqlalchemy.orm import relationship
from sickrage.core import common
from sickrage.core.common import SearchFormats
from sickrage.core.databases import SRDatabase, SRDatabaseBase
......@@ -31,7 +32,7 @@ class MainDBBase(SRDatabaseBase):
class MainDB(SRDatabase):
def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
super(MainDB, self).__init__('main', 11, db_type, db_prefix, db_host, db_port, db_username, db_password)
super(MainDB, self).__init__('main', 12, db_type, db_prefix, db_host, db_port, db_username, db_password)
MainDBBase.metadata.create_all(self.engine)
for model in MainDBBase._decl_class_registry.values():
if hasattr(model, '__tablename__'):
......@@ -54,10 +55,8 @@ class MainDB(SRDatabase):
status = Column(Text, default='')
flatten_folders = Column(Boolean, default=0)
paused = Column(Boolean, default=0)
air_by_date = Column(Boolean, default=0)
search_format = Column(Integer, default=SearchFormats.STANDARD)
anime = Column(Boolean, default=0)
scene = Column(Boolean, default=0)
sports = Column(Boolean, default=0)
subtitles = Column(Boolean, default=0)
dvdorder = Column(Boolean, default=0)
skip_downloaded = Column(Boolean, default=0)
......
import datetime
from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sickrage.core import common
from sickrage.core.common import SearchFormats
Base = declarative_base()
def upgrade(migrate_engine):
meta = MetaData(migrate_engine)
tv_shows = Table('tv_shows', meta, autoload=True)
if not hasattr(tv_shows.c, 'search_format'):
search_format = Column('search_format', Integer, default=0)
search_format.create(tv_shows)
with migrate_engine.begin() as conn:
for row in migrate_engine.execute(tv_shows.select()):
if row.anime == 1 and not row.scene == 1:
value = SearchFormats.ANIME
elif row.anime == 1 and row.scene == 1:
value = SearchFormats.SCENE
elif row.sports == 1:
value = SearchFormats.SPORTS
elif row.air_by_date == 1:
value = SearchFormats.AIR_BY_DATE
elif row.scene == 1:
value = SearchFormats.SCENE
else:
value = SearchFormats.STANDARD
conn.execute(tv_shows.update().where(tv_shows.c.indexer_id == row.indexer_id).values(search_format=value))
class TVShowBackup(Base):
__tablename__ = 'tv_shows_backup'
indexer_id = Column(Integer, index=True, primary_key=True)
indexer = Column(Integer, index=True, primary_key=True)
name = Column(Text, default='')
location = Column(Text, default='')
network = Column(Text, default='')
genre = Column(Text, default='')
overview = Column(Text, default='')
classification = Column(Text, default='Scripted')
runtime = Column(Integer, default=0)
quality = Column(Integer, default=-1)
airs = Column(Text, default='')
status = Column(Text, default='')
flatten_folders = Column(Boolean, default=0)
paused = Column(Boolean, default=0)
search_format = Column(Integer, default=SearchFormats.STANDARD)
anime = Column(Boolean, default=0)
subtitles = Column(Boolean, default=0)
dvdorder = Column(Boolean, default=0)
skip_downloaded = Column(Boolean, default=0)
startyear = Column(Integer, default=0)
lang = Column(Text, default='')
imdb_id = Column(Text, default='')
rls_ignore_words = Column(Text, default='')
rls_require_words = Column(Text, default='')
default_ep_status = Column(Integer, default=common.SKIPPED)
sub_use_sr_metadata = Column(Boolean, default=0)
notify_list = Column(Text, default='')
search_delay = Column(Integer, default=0)
scene_exceptions = Column(Text, default='')
last_scene_exceptions_refresh = Column(Integer, default=0)
last_update = Column(Integer, default=datetime.datetime.now().toordinal())
last_refresh = Column(Integer, default=datetime.datetime.now().toordinal())
last_backlog_search = Column(Integer, default=0)
last_proper_search = Column(Integer, default=0)
TVShowBackup.__table__.create(migrate_engine)
tv_shows_backup = Table('tv_shows_backup', meta, autoload=True)
session = sessionmaker(bind=migrate_engine)()
for row in session.query(tv_shows):
tv_shows_backup.insert().execute(dict((k, getattr(row, k)) for k in row.keys() if k not in ['scene', 'sports', 'air_by_date']))
session.commit()
session.close()
tv_shows.drop()
tv_shows_backup.rename('tv_shows')
def downgrade(migrate_engine):
pass
......@@ -28,6 +28,7 @@ from functools import partial
import sickrage
from sickrage.core import common
from sickrage.core.common import SearchFormats
from sickrage.core.helpers import sanitize_scene_name, strip_accents
from sickrage.core.tv.show.helpers import find_show
......@@ -169,10 +170,10 @@ def make_scene_season_search_string(show_id, season, episode, extraSearchType=No
show_object = find_show(show_id)
episode_object = show_object.get_episode(season, episode)
if show_object.air_by_date or show_object.sports:
if show_object.search_format in [SearchFormats.AIR_BY_DATE, SearchFormats.SPORTS]:
# the search string for air by date shows is just
seasonStrings = [str(episode_object.airdate).split('-')[0]]
elif show_object.is_anime:
elif show_object.search_format == SearchFormats.ANIME:
# get show qualities
anyQualities, bestQualities = common.Quality.split_quality(show_object.quality)
......@@ -196,7 +197,6 @@ def make_scene_season_search_string(show_id, season, episode, extraSearchType=No
ab_number = episode.scene_absolute_number
if ab_number > 0:
seasonStrings.append("%02d" % ab_number)
else:
numseasons = len(set([s.season for s in show_object.episodes if s.season > 0]))
seasonStrings = ["S%02d" % int(episode_object.scene_season)]
......@@ -235,9 +235,9 @@ def make_scene_search_string(show_id, season, episode):
numseasons = len(set([s.season for s in show_object.episodes if s.season > 0]))
# see if we should use dates instead of episodes
if (show_object.air_by_date or show_object.sports) and show_object.airdate > datetime.date.min:
if show_object.search_format in [SearchFormats.AIR_BY_DATE, SearchFormats.SPORTS] and show_object.airdate > datetime.date.min:
epStrings = [str(show_object.airdate)]
elif show_object.is_anime:
elif show_object.search_format == SearchFormats.ANIME:
epStrings = ["%02i" % int(show_object.scene_absolute_number if show_object.scene_absolute_number > 0 else show_object.scene_episode)]
else:
epStrings = ["S%02iE%02i" % (int(show_object.scene_season), int(show_object.scene_episode)),
......
......@@ -31,6 +31,7 @@ from sqlalchemy import orm
import sickrage
from sickrage.core import common
from sickrage.core.common import SearchFormats
from sickrage.core.databases.main import MainDB
from sickrage.core.helpers import remove_extension, strip_accents
from sickrage.core.nameparser import regexes
......@@ -160,10 +161,6 @@ class NameParser(object):
result.series_name = match.group('series_name')
if result.series_name:
result.series_name = self.clean_series_name(result.series_name)
result.score += 1
if 'series_num' in named_groups and match.group('series_num'):
result.score += 1
if 'season_num' in named_groups:
tmp_season = int(match.group('season_num'))
......@@ -173,7 +170,6 @@ class NameParser(object):
continue
result.season_number = tmp_season
result.score += 1
if 'ep_num' in named_groups:
ep_num = self._convert_number(match.group('ep_num'))
......@@ -185,16 +181,12 @@ class NameParser(object):
tmp_episodes = [ep_num]
result.episode_numbers = tmp_episodes
result.score += 3
if 'ep_ab_num' in named_groups:
ep_ab_num = self._convert_number(match.group('ep_ab_num'))
result.score += 1
if 'extra_ab_ep_num' in named_groups and match.group('extra_ab_ep_num'):
result.ab_episode_numbers = list(range(ep_ab_num,
self._convert_number(match.group('extra_ab_ep_num')) + 1))
result.score += 1
result.ab_episode_numbers = list(range(ep_ab_num, self._convert_number(match.group('extra_ab_ep_num')) + 1))
else:
result.ab_episode_numbers = [ep_ab_num]
......@@ -202,7 +194,6 @@ class NameParser(object):
air_date = match.group('air_date')
try:
result.air_date = parser.parse(air_date, fuzzy=True).date()
result.score += 1
except Exception:
continue
......@@ -214,11 +205,9 @@ class NameParser(object):
r'([. _-]|^)(special|extra)s?\w*([. _-]|$)', tmp_extra_info, re.I):
continue
result.extra_info = tmp_extra_info
result.score += 1
if 'release_group' in named_groups:
result.release_group = match.group('release_group')
result.score += 1
if 'version' in named_groups:
# assigns version to anime file if detected using anime regex. Non-anime regex receives -1
......@@ -230,6 +219,7 @@ class NameParser(object):
else:
result.version = -1
result.score += len([x for x in result.__dict__ if getattr(result, x, None) is not None])
matches.append(result)
if len(matches):
......@@ -291,7 +281,7 @@ class NameParser(object):
s = season_number
e = epNo
if show_obj.is_scene and not skip_scene_detection:
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
(s, e) = get_indexer_numbering(show_obj.indexer_id,
show_obj.indexer,
season_number,
......@@ -303,7 +293,7 @@ class NameParser(object):
for epAbsNo in best_result.ab_episode_numbers:
a = epAbsNo
if show_obj.is_scene:
if show_obj.search_format == SearchFormats.SCENE:
scene_result = show_obj.get_scene_exception_by_name(best_result.series_name)
if scene_result:
a = get_indexer_absolute_numbering(show_obj.indexer_id,
......@@ -321,7 +311,7 @@ class NameParser(object):
s = best_result.season_number
e = epNo
if show_obj.is_scene and not skip_scene_detection:
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
(s, e) = get_indexer_numbering(show_obj.indexer_id,
show_obj.indexer,
best_result.season_number,
......@@ -360,7 +350,7 @@ class NameParser(object):
best_result.episode_numbers = new_episode_numbers
best_result.season_number = new_season_numbers[0]
if show_obj.is_scene and not skip_scene_detection:
if show_obj.search_format == SearchFormats.SCENE and not skip_scene_detection:
sickrage.app.log.debug("Scene converted parsed result {} into {}".format(best_result.original_name, best_result))
# CPU sleep
......
......@@ -192,6 +192,18 @@ normal_regexes = [
([. _-]+(?P<extra_info>(?!\d{3}[. _-]+)[^-]+) # Source_Quality_Etc-
(-(?P<release_group>[^ -]+([. _-]\[.*\])?))?)?$ # Group
'''),