Commit 4d28e144 authored by echel0n's avatar echel0n
Browse files

Merge branch 'release/9.2.62'

parents d79c6032 20416e1b
# Changelog
- * a264d26 - 2017-12-10: Release v9.2.61
- * fbfdeb6 - 2017-12-13: Release v9.2.62
- * 34d3b93 - 2017-12-13: Added code to name cache to allow adding show names we wish to ignore, helps speed up name parsing and searches
- * b922499 - 2017-12-12: Py3 compat for queue
- * 2c6c207 - 2017-12-11: Fixed issue with thread naming for post-processing
- * 5c4b822 - 2017-12-11: refactored remaining GET database calls
- * c07404b - 2017-12-11: Fixed issues with SkyTorrents
- * 8923500 - 2017-12-11: Fixed issues with failed and cache database md5 checksums
- * 93c5ad5 - 2017-12-11: Fixed issues with failed and cache database cleanup on new revs
- * 8e43de8 - 2017-12-11: Fixed issues with removing duplicate shows and episodes
- * 7b19a3c - 2017-12-11: Updated misc provider code
- * 3f23fd2 - 2017-12-11: refactored remaining database calls to use custom database calls
- * cf0821c - 2017-12-11: Fixed issue #151 - using unicode instead of str to result post-process results
- * 4323d84 - 2017-12-10: Refactored database calls, resolves memory usage issues
- * 0d8b4d1 - 2017-12-10: Small memory footprint improvement
- * f2d7dc6 - 2017-12-10: Release v9.2.61
- * 656a038 - 2017-12-10: Overall stats now only displayed for main shows page, helps reduce overhead
- * 75fa296 - 2017-12-10: Release v9.2.57
- * 6c4534b - 2017-12-10: Fixed TorrentRSSFeed issues
......
......@@ -532,7 +532,7 @@ class Core(object):
Populates the showlist with shows from the database
"""
for dbData in [x['doc'] for x in self.main_db.db.all('tv_shows', with_doc=True)]:
for dbData in self.main_db.all('tv_shows'):
try:
self.log.debug("Loading data for show: [{}]".format(dbData['show_name']))
self.showlist += [TVShow(int(dbData['indexer']), int(dbData['indexer_id']))]
......
......@@ -20,9 +20,9 @@
from __future__ import unicode_literals
from CodernityDB.database import RecordNotFound
from adba.aniDBerrors import AniDBCommandTimeoutError
import sickrage
from adba.aniDBerrors import AniDBCommandTimeoutError
class BlackAndWhiteList(object):
......@@ -51,7 +51,7 @@ class BlackAndWhiteList(object):
:param values: Values to be inserted in table
"""
for value in values:
sickrage.app.main_db.db.insert({
sickrage.app.main_db.insert({
'_t': table,
'show_id': self.show_id,
'keywork': value
......@@ -86,7 +86,7 @@ class BlackAndWhiteList(object):
:param table: database table remove keywords from
"""
try:
sickrage.app.main_db.db.delete(sickrage.app.main_db.db.get(table, self.show_id, with_doc=True)['doc'])
sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
except RecordNotFound:
pass
......@@ -98,9 +98,7 @@ class BlackAndWhiteList(object):
:return: keywords in list
"""
groups = []
for result in [x['doc'] for x in sickrage.app.main_db.db.get_many(table, self.show_id, with_doc=True)]:
groups.append(result['keyword'])
groups = [x['keyword']for x in sickrage.app.main_db.get_many(table, self.show_id)]
sickrage.app.log.debug(
'BWL: ' + str(self.show_id) + ' loaded keywords from ' + table + ': ' + str(groups))
......
......@@ -43,7 +43,7 @@ class NameCache(object):
def put(self, name, indexer_id=0):
"""
Adds the show & tvdb id to the scene_names table in cache.db.
Adds the show & tvdb id to the scene_names table in cache db
:param name: The show name to cache
:param indexer_id: the TVDB id that this show should be cached with (can be None/0 for unknown)
......@@ -51,29 +51,29 @@ class NameCache(object):
# standardize the name we're using to account for small differences in providers
name = full_sanitizeSceneName(name)
if name not in self.cache:
self.cache[name] = int(indexer_id)
try:
if not len([x['doc'] for x in sickrage.app.cache_db.db.get_many('scene_names', name, with_doc=True) if
x['doc']['indexer_id'] == indexer_id]):
# insert name into cache
sickrage.app.cache_db.db.insert({
'_t': 'scene_names',
'indexer_id': indexer_id,
'name': name
})
except RecordNotFound:
self.cache[name] = int(indexer_id)
try:
if not len([x for x in sickrage.app.cache_db.get_many('scene_names', name)
if x['indexer_id'] == indexer_id]):
# insert name into cache
sickrage.app.cache_db.db.insert({
sickrage.app.cache_db.insert({
'_t': 'scene_names',
'indexer_id': indexer_id,
'name': name
})
except RecordNotFound:
# insert name into cache
sickrage.app.cache_db.insert({
'_t': 'scene_names',
'indexer_id': indexer_id,
'name': name
})
def get(self, name):
"""
Looks up the given name in the scene_names table in cache.db.
Looks up the given name in the scene_names table in cache db
:param name: The show name to look up.
:return: the TVDB id that resulted from the cache lookup or None if the show wasn't found in the cache
......@@ -82,33 +82,32 @@ class NameCache(object):
if name in self.cache:
return int(self.cache[name])
def clear(self, indexerid=0):
def clear(self, indexerid):
"""
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
Deletes all entries from the cache matching the indexerid.
"""
[sickrage.app.cache_db.db.delete(x['doc']) for x in
sickrage.app.cache_db.db.all('scene_names', with_doc=True)
if x['doc']['indexer_id'] in [indexerid, 0]]
[sickrage.app.cache_db.delete(x) for x in
sickrage.app.cache_db.all('scene_names')
if x['indexer_id'] == indexerid]
for item in [self.cache[key] for key, value in self.cache.items() if value == 0 or value == indexerid]:
for item in [self.cache[key] for key, value in self.cache.items() if value == indexerid]:
del item
def load(self):
self.cache = dict([(x['doc']['name'], x['doc']['indexer_id']) for x in
sickrage.app.cache_db.db.all('scene_names', with_doc=True)])
self.cache = dict([(x['name'], x['indexer_id']) for x in sickrage.app.cache_db.all('scene_names')])
def save(self):
"""Commit cache to database file"""
for name, indexer_id in self.cache.items():
try:
if len([x['doc'] for x in sickrage.app.cache_db.db.get_many('scene_names', name, with_doc=True) if
x['doc']['indexer_id'] == indexer_id]):
if len([x for x in sickrage.app.cache_db.get_many('scene_names', name)
if x['indexer_id'] == indexer_id]):
continue
except RecordNotFound:
pass
# insert name into cache
sickrage.app.cache_db.db.insert({
sickrage.app.cache_db.insert({
'_t': 'scene_names',
'indexer_id': indexer_id,
'name': name
......
......@@ -41,8 +41,8 @@ class TVCache(object):
def clear(self):
if self.shouldClearCache():
[sickrage.app.cache_db.db.delete(x['doc']) for x in
sickrage.app.cache_db.db.get_many('providers', self.providerID, with_doc=True)]
[sickrage.app.cache_db.delete(x) for x in
sickrage.app.cache_db.get_many('providers', self.providerID)]
def _get_title_and_url(self, item):
return self.provider._get_title_and_url(item)
......@@ -120,7 +120,7 @@ class TVCache(object):
@property
def last_update(self):
try:
dbData = sickrage.app.cache_db.db.get('lastUpdate', self.providerID, with_doc=True)['doc']
dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
lastTime = int(dbData["time"])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): lastTime = 0
except RecordNotFound:
......@@ -131,11 +131,11 @@ class TVCache(object):
@last_update.setter
def last_update(self, toDate):
try:
dbData = sickrage.app.cache_db.db.get('lastUpdate', self.providerID, with_doc=True)['doc']
dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
dbData['time'] = int(time.mktime(toDate.timetuple()))
sickrage.app.cache_db.db.update(dbData)
sickrage.app.cache_db.update(dbData)
except RecordNotFound:
sickrage.app.cache_db.db.insert({
sickrage.app.cache_db.insert({
'_t': 'lastUpdate',
'provider': self.providerID,
'time': int(time.mktime(toDate.timetuple()))
......@@ -144,7 +144,7 @@ class TVCache(object):
@property
def last_search(self):
try:
dbData = sickrage.app.cache_db.db.get('lastSearch', self.providerID, with_doc=True)['doc']
dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
lastTime = int(dbData["time"])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): lastTime = 0
except RecordNotFound:
......@@ -155,11 +155,11 @@ class TVCache(object):
@last_search.setter
def last_search(self, toDate):
try:
dbData = sickrage.app.cache_db.db.get('lastSearch', self.providerID, with_doc=True)['doc']
dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
dbData['time'] = int(time.mktime(toDate.timetuple()))
sickrage.app.cache_db.db.update(dbData)
sickrage.app.cache_db.update(dbData)
except RecordNotFound:
sickrage.app.cache_db.db.insert({
sickrage.app.cache_db.insert({
'_t': 'lastUpdate',
'provider': self.providerID,
'time': int(time.mktime(toDate.timetuple()))
......@@ -181,8 +181,8 @@ class TVCache(object):
def addCacheEntry(self, name, url, seeders, leechers, size):
# check for existing entry in cache
if len([x for x in sickrage.app.cache_db.db.get_many('providers', self.providerID, with_doc=True) if
x['doc']['url'] == url]): return
if len([x for x in sickrage.app.cache_db.get_many('providers', self.providerID)
if x['url'] == url]): return
try:
# parse release name
......@@ -222,7 +222,7 @@ class TVCache(object):
}
# add to internal database
sickrage.app.cache_db.db.insert(dbData)
sickrage.app.cache_db.insert(dbData)
# add to external database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
......@@ -247,8 +247,9 @@ class TVCache(object):
pass
# get data from internal database
dbData += [x['doc'] for x in sickrage.app.cache_db.db.get_many('providers', self.providerID, with_doc=True)]
dbData += [x for x in sickrage.app.cache_db.get_many('providers', self.providerID)]
# sort data by criteria
dbData = [x for x in dbData if x['indexerid'] == ep_obj.show.indexerid and x['season'] == ep_obj.season
and "|" + str(ep_obj.episode) + "|" in x['episodes']] if ep_obj else dbData
......
......@@ -207,9 +207,9 @@ class srDatabase(object):
try:
for x in self.db.all(index_name):
try:
self.db.get('id', x.get('_id'), with_doc=True)
self.get('id', x.get('_id'))
except (ValueError, TypeError) as e:
self.db.delete(self.db.get(index_name, x.get('key'), with_doc=True)['doc'])
self.delete(self.get(index_name, x.get('key')))
except Exception as e:
if index_name in self.db.indexes_names:
self.db.destroy_index(self.db.indexes_names[index_name])
......@@ -265,10 +265,10 @@ class srDatabase(object):
if isinstance(v, list):
for d in v:
d.update({'_t': t_name})
self.db.insert(d)
self.insert(d)
else:
v.update({'_t': t_name})
self.db.insert(v)
self.insert(v)
sickrage.app.log.info('Total migration took %s', (time.time() - migrate_start))
sickrage.app.log.info('=' * 30)
......@@ -293,6 +293,27 @@ class srDatabase(object):
if os.path.isfile(self.old_db_path + '-shm'):
os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random))
def all(self, *args, **kwargs):
kwargs['with_doc'] = True
return (x['doc'] for x in self.db.all(*args, **kwargs))
def get_many(self, *args, **kwargs):
kwargs['with_doc'] = True
return (x['doc'] for x in self.db.get_many(*args, **kwargs))
def get(self, *args, **kwargs):
kwargs['with_doc'] = True
data = self.db.get(*args, **kwargs)
return data.get('doc', data)
def delete(self, *args):
return self.db.delete(*args)
def update(self, *args):
return self.db.update(*args)
def insert(self, *args):
return self.db.insert(*args)
# Monkey-Patch storage to suppress logging messages
IU_Storage.get = Custom_IU_Storage_get
......@@ -48,11 +48,4 @@ class CacheDB(srDatabase):
def __init__(self, name='cache'):
super(CacheDB, self).__init__(name)
self.old_db_path = os.path.join(sickrage.app.data_dir, 'cache.db')
def check_versions(self, index_name, current_version, previous_version):
# Wipe table if versions are different
if previous_version < current_version:
for x in self.db.all(index_name, with_doc=True):
self.db.delete(x['doc'])
super(CacheDB, self).check_versions(index_name, current_version, previous_version)
self.old_db_path = os.path.join(sickrage.app.data_dir, 'cache.db')
\ No newline at end of file
......@@ -24,7 +24,7 @@ from CodernityDB.hash_index import HashIndex
class CacheLastUpdateIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......@@ -39,7 +39,7 @@ class CacheLastUpdateIndex(HashIndex):
class CacheLastSearchIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......@@ -69,7 +69,7 @@ class CacheSceneExceptionsIndex(HashIndex):
class CacheSceneNamesIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......@@ -84,7 +84,7 @@ class CacheSceneNamesIndex(HashIndex):
class CacheNetworkTimezonesIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......@@ -99,7 +99,7 @@ class CacheNetworkTimezonesIndex(HashIndex):
class CacheSceneExceptionsRefreshIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......@@ -114,7 +114,7 @@ class CacheSceneExceptionsRefreshIndex(HashIndex):
class CacheProvidersIndex(HashIndex):
_version = 3
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......
......@@ -38,11 +38,4 @@ class FailedDB(srDatabase):
def __init__(self, name='failed'):
super(FailedDB, self).__init__(name)
self.old_db_path = os.path.join(sickrage.app.data_dir, 'failed.db')
def check_versions(self, index_name, current_version, previous_version):
# Wipe table if versions are different
if previous_version < current_version:
for x in self.db.all(index_name, with_doc=True):
self.db.delete(x['doc'])
super(FailedDB, self).check_versions(index_name, current_version, previous_version)
\ No newline at end of file
self.old_db_path = os.path.join(sickrage.app.data_dir, 'failed.db')
\ No newline at end of file
......@@ -24,7 +24,7 @@ from CodernityDB.hash_index import HashIndex
class FailedIndex(HashIndex):
_version = 2
_version = 4
def __init__(self, *args, **kwargs):
kwargs['key_format'] = '32s'
......
......@@ -77,7 +77,7 @@ class MainDB(srDatabase):
def fix_show_none_types(self):
checked = []
for show in [x['doc'] for x in self.db.all('tv_shows', with_doc=True)]:
for show in self.all('tv_shows'):
if show['indexer_id'] in checked:
continue
......@@ -91,14 +91,16 @@ class MainDB(srDatabase):
pass
if dirty:
self.db.update(show)
self.update(show)
checked += [show['indexer_id']]
del checked
def fix_episode_none_types(self):
checked = []
for ep in [x['doc'] for x in self.db.all('tv_episodes', with_doc=True)]:
for ep in self.all('tv_episodes'):
if ep['showid'] in checked:
continue
......@@ -112,39 +114,36 @@ class MainDB(srDatabase):
pass
if dirty:
self.db.update(ep)
self.update(ep)
checked += [ep['showid']]
def fix_dupe_shows(self):
checked = []
del checked
for show in [x['doc'] for x in self.db.all('tv_shows', with_doc=True)]:
if show['indexer_id'] in checked:
continue
def fix_dupe_shows(self):
found = []
for dupe in [x['doc'] for x in self.db.get_many('tv_shows', show['indexer_id'], with_doc=True)][1::]:
sickrage.app.log.info("Deleting duplicate show with id: {}".format(dupe["indexer_id"]))
self.db.delete(dupe)
for show in self.all('tv_shows'):
if show['indexer_id'] in found:
sickrage.app.log.info("Deleting duplicate show with id: {}".format(show["indexer_id"]))
self.delete(show)
found += [show['indexer_id']]
checked += [show['indexer_id']]
del found
def fix_dupe_episodes(self):
checked = []
for ep in [x['doc'] for x in self.db.all('tv_episodes', with_doc=True)]:
if ep['showid'] in checked:
continue
found = []
for dupe in [x['doc'] for x in self.db.get_many('tv_episodes', ep['showid'], with_doc=True) if
x['doc']['indexerid'] == ep['indexerid']][1::]:
sickrage.app.log.info("Deleting duplicate episode with id: {}".format(dupe["indexerid"]))
self.db.delete(dupe)
for ep in self.all('tv_episodes'):
if ep['indexerid'] in found:
sickrage.app.log.info("Deleting duplicate episode with id: {}".format(ep["indexerid"]))
self.delete(ep)
found += [ep['indexerid']]
checked += [ep['showid']]
del found
def fix_orphaned_episodes(self):
for ep in [x['doc'] for x in self.db.all('tv_episodes', with_doc=True)]:
if not self.db.get('tv_shows', ep['showid'], with_doc=True)['doc']:
for ep in self.all('tv_episodes'):
if not self.get('tv_shows', ep['showid']):
sickrage.app.log.info("Deleting orphan episode with id: {}".format(ep["indexerid"]))
self.db.delete(ep)
self.delete(ep)
......@@ -1549,7 +1549,7 @@ def overall_stats():
'total_size': 0
}
for result in [x['doc'] for x in sickrage.app.main_db.db.all('tv_episodes', with_doc=True)]:
for result in sickrage.app.main_db.all('tv_episodes'):
if not (result['season'] > 0 and result['episode'] > 0 and result['airdate'] > 1):
continue
......
......@@ -196,9 +196,8 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
seasonStrings.append("%02d" % ab_number)
else:
numseasons = len({x['doc']['season'] for x in
sickrage.app.main_db.db.get_many('tv_episodes', show.indexerid, with_doc=True)
if x['doc']['season'] != 0})
numseasons = len({x['season'] for x in sickrage.app.main_db.get_many('tv_episodes', show.indexerid)
if x['season'] != 0})
seasonStrings = ["S%02d" % int(ep_obj.scene_season)]
......@@ -230,9 +229,8 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
def makeSceneSearchString(show, ep_obj):
toReturn = []
numseasons = len(
{x['doc']['season'] for x in sickrage.app.main_db.db.get_many('tv_episodes', show.indexerid, with_doc=True)
if x['doc']['season'] != 0})
numseasons = len({x['season'] for x in sickrage.app.main_db.get_many('tv_episodes', show.indexerid)
if x['season'] != 0})
# see if we should use dates instead of episodes
if (show.air_by_date or show.sports) and ep_obj.airdate != date.fromordinal(1):
......
......@@ -58,7 +58,7 @@ class NameParser(object):
def get_show(self, name):
show = None
show_id = 0
show_id = None
if not all([name, sickrage.app.showlist]):
return show, show_id
......@@ -82,19 +82,24 @@ class NameParser(object):
# lookup show id
for lookup in lookup_list:
if show:
if show or show_id is not None:
continue
try:
from sickrage.core.tv.show import TVShow
show_id = int(lookup())
sickrage.app.name_cache.put(name, show_id)
if self.validate_show:
show = findCertainShow(int(lookup()))
show = findCertainShow(show_id)
else:
show = TVShow(1, int(lookup()))
sickrage.app.name_cache.put(name, show.indexerid)
from sickrage.core.tv.show import TVShow
show = TVShow(1, show_id)
except Exception:
pass
if show_id is None:
# ignore show name by caching it with a indexerid of 0
sickrage.app.name_cache.put(name, 0)
return show, show_id or 0
@staticmethod
......@@ -268,9 +273,8 @@ class NameParser(object):
if bestResult.is_air_by_date:
airdate = bestResult.air_date.toordinal()
dbData = [x['doc'] for x in
sickrage.app.main_db.db.get_many('tv_episodes', bestResult.show.indexerid, with_doc=True)
if x['doc']['indexer'] == bestResult.show.indexer and x['doc']['airdate'] == airdate]
dbData = [x for x in sickrage.app.main_db.get_many('tv_episodes', bestResult.show.indexerid)
if x['indexer'] == bestResult.show.indexer and x['airdate'] == airdate]
season_number = None
episode_numbers = []
......
......@@ -47,9 +47,6 @@ class ProcessResult(object):
def __unicode__(self):
return self.output
def __str__(self):
return self.__unicode__().encode('utf-8', errors='replace')
def delete_folder(folder, check_empty=True):
"""
Removes a folder from the filesystem
......@@ -307,7 +304,7 @@ def validateDir(process_path, release_name, failed, result):
# make sure the dir isn't inside a show dir
for show in sickrage.app.showlist:
if process_path.lower().startswith(os.path.realpath(show.location).lower() + os.sep) or \