Commit 007df0a3 authored by echel0n's avatar echel0n
Browse files

Refactored get, all, and get_many database functions.

parent e0edcfe8
# Changelog # Changelog
- * c1ad198 - 2018-11-11: Release v9.4.29 - * 0e4c60f - 2018-11-11: Refactored get, all, and get_many database functions.
- * f87d6ad - 2018-11-11: Release v9.4.29
- * 7d0812a - 2018-11-11: Pre-Release v9.4.29.dev2 - * 7d0812a - 2018-11-11: Pre-Release v9.4.29.dev2
- * 8028ae9 - 2018-11-11: Default subtitle language of English choosen if no language is specified in subtitle settings - * 8028ae9 - 2018-11-11: Default subtitle language of English choosen if no language is specified in subtitle settings
- * 5eef56e - 2018-11-11: Pre-Release v9.4.29.dev1 - * 5eef56e - 2018-11-11: Pre-Release v9.4.29.dev1
......
...@@ -19,8 +19,6 @@ ...@@ -19,8 +19,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from adba.aniDBerrors import AniDBCommandTimeoutError from adba.aniDBerrors import AniDBCommandTimeoutError
...@@ -85,10 +83,8 @@ class BlackAndWhiteList(object): ...@@ -85,10 +83,8 @@ class BlackAndWhiteList(object):
:param table: database table remove keywords from :param table: database table remove keywords from
""" """
try: sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
except RecordNotFound:
pass
def _load_list(self, table): def _load_list(self, table):
""" """
......
...@@ -22,8 +22,6 @@ from __future__ import unicode_literals ...@@ -22,8 +22,6 @@ from __future__ import unicode_literals
import time import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from sickrage.core.helpers import full_sanitizeSceneName from sickrage.core.helpers import full_sanitizeSceneName
from sickrage.core.helpers.encoding import strip_accents from sickrage.core.helpers.encoding import strip_accents
...@@ -55,16 +53,8 @@ class NameCache(object): ...@@ -55,16 +53,8 @@ class NameCache(object):
self.cache[name] = int(indexer_id) self.cache[name] = int(indexer_id)
try: dbData = [x for x in sickrage.app.cache_db.get_many('scene_names', name) if x['indexer_id'] == indexer_id]
if not len([x for x in sickrage.app.cache_db.get_many('scene_names', name) if not len(dbData):
if x['indexer_id'] == indexer_id]):
# insert name into cache
sickrage.app.cache_db.insert({
'_t': 'scene_names',
'indexer_id': indexer_id,
'name': name
})
except RecordNotFound:
# insert name into cache # insert name into cache
sickrage.app.cache_db.insert({ sickrage.app.cache_db.insert({
'_t': 'scene_names', '_t': 'scene_names',
...@@ -102,12 +92,9 @@ class NameCache(object): ...@@ -102,12 +92,9 @@ class NameCache(object):
def save(self): def save(self):
"""Commit cache to database file""" """Commit cache to database file"""
for name, indexer_id in self.cache.items(): for name, indexer_id in self.cache.items():
try: dbData = [x for x in sickrage.app.cache_db.get_many('scene_names', name) if x['indexer_id'] == indexer_id]
if len([x for x in sickrage.app.cache_db.get_many('scene_names', name) if len(dbData):
if x['indexer_id'] == indexer_id]): continue
continue
except RecordNotFound:
pass
# insert name into cache # insert name into cache
sickrage.app.cache_db.insert({ sickrage.app.cache_db.insert({
......
...@@ -22,8 +22,6 @@ import datetime ...@@ -22,8 +22,6 @@ import datetime
import time import time
import feedparser import feedparser
from CodernityDB.database import RecordNotFound
from CodernityDB.index import IndexNotFoundException
import sickrage import sickrage
from sickrage.core.api.cache import ProviderCacheAPI from sickrage.core.api.cache import ProviderCacheAPI
...@@ -121,23 +119,23 @@ class TVCache(object): ...@@ -121,23 +119,23 @@ class TVCache(object):
@property @property
def last_update(self): def last_update(self):
try: dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID) if dbData:
lastTime = int(dbData["time"]) lastTime = int(dbData["time"])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
lastTime = 0 lastTime = 0
except (RecordNotFound, IndexNotFoundException): else:
lastTime = 0 lastTime = 0
return datetime.datetime.fromtimestamp(lastTime) return datetime.datetime.fromtimestamp(lastTime)
@last_update.setter @last_update.setter
def last_update(self, toDate): def last_update(self, toDate):
try: dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID) if dbData:
dbData['time'] = int(time.mktime(toDate.timetuple())) dbData['time'] = int(time.mktime(toDate.timetuple()))
sickrage.app.cache_db.update(dbData) sickrage.app.cache_db.update(dbData)
except (RecordNotFound, IndexNotFoundException): else:
sickrage.app.cache_db.insert({ sickrage.app.cache_db.insert({
'_t': 'lastUpdate', '_t': 'lastUpdate',
'provider': self.providerID, 'provider': self.providerID,
...@@ -146,23 +144,23 @@ class TVCache(object): ...@@ -146,23 +144,23 @@ class TVCache(object):
@property @property
def last_search(self): def last_search(self):
try: dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
dbData = sickrage.app.cache_db.get('lastSearch', self.providerID) if dbData:
lastTime = int(dbData["time"]) lastTime = int(dbData["time"])
if lastTime > int(time.mktime(datetime.datetime.today().timetuple())): if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
lastTime = 0 lastTime = 0
except (RecordNotFound, IndexNotFoundException): else:
lastTime = 0 lastTime = 0
return datetime.datetime.fromtimestamp(lastTime) return datetime.datetime.fromtimestamp(lastTime)
@last_search.setter @last_search.setter
def last_search(self, toDate): def last_search(self, toDate):
try: dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
dbData = sickrage.app.cache_db.get('lastSearch', self.providerID) if dbData:
dbData['time'] = int(time.mktime(toDate.timetuple())) dbData['time'] = int(time.mktime(toDate.timetuple()))
sickrage.app.cache_db.update(dbData) sickrage.app.cache_db.update(dbData)
except (RecordNotFound, IndexNotFoundException): else:
sickrage.app.cache_db.insert({ sickrage.app.cache_db.insert({
'_t': 'lastUpdate', '_t': 'lastUpdate',
'provider': self.providerID, 'provider': self.providerID,
......
...@@ -26,6 +26,7 @@ import time ...@@ -26,6 +26,7 @@ import time
import traceback import traceback
from sqlite3 import OperationalError from sqlite3 import OperationalError
from CodernityDB.database import RecordDeleted, RecordNotFound
from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
from CodernityDB.index import IndexNotFoundException, IndexConflict, IndexException from CodernityDB.index import IndexNotFoundException, IndexConflict, IndexException
from CodernityDB.storage import IU_Storage from CodernityDB.storage import IU_Storage
...@@ -334,15 +335,46 @@ class srDatabase(object): ...@@ -334,15 +335,46 @@ class srDatabase(object):
if os.path.isfile(self.old_db_path + '-shm'): if os.path.isfile(self.old_db_path + '-shm'):
os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random)) os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random))
def delete_corrupted(self, _id, traceback_error=''):
try:
sickrage.app.log.debug('Deleted corrupted document "{}": {}'.format(_id, traceback_error))
corrupted = self.db.get('id', _id, with_storage=False)
self.db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
except:
log.debug('Failed deleting corrupted: {}'.format(traceback.format_exc()))
def all(self, *args, **kwargs): def all(self, *args, **kwargs):
return (x['doc'] for x in self.db.all(with_doc=True, *args, **kwargs)) for data in self.db.all(*args, **kwargs):
if kwargs.pop('with_doc', True):
try:
doc = self.db.get('id', data['_id'])
yield doc
except (RecordDeleted, RecordNotFound):
sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
except (ValueError, EOFError):
self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
else:
yield data
def get_many(self, *args, **kwargs): def get_many(self, *args, **kwargs):
return (x['doc'] for x in self.db.get_many(with_doc=True, *args, **kwargs)) for data in self.db.get_many(*args, **kwargs):
if kwargs.pop('with_doc', True):
try:
doc = self.db.get('id', data['_id'])
yield doc
except (RecordDeleted, RecordNotFound):
sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
except (ValueError, EOFError):
self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
else:
yield data
def get(self, *args, **kwargs): def get(self, *args, **kwargs):
x = self.db.get(with_doc=True, *args, **kwargs) try:
return x.get('doc', x) x = self.db.get(with_doc=kwargs.pop('with_doc', True), *args, **kwargs)
return x.get('doc', x)
except (RecordDeleted, RecordNotFound):
pass
def delete(self, *args): def delete(self, *args):
return self.db.delete(*args) return self.db.delete(*args)
......
...@@ -24,8 +24,6 @@ import re ...@@ -24,8 +24,6 @@ import re
import threading import threading
import time import time
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from adba.aniDBAbstracter import Anime from adba.aniDBAbstracter import Anime
from sickrage.core.helpers import full_sanitizeSceneName, sanitizeSceneName from sickrage.core.helpers import full_sanitizeSceneName, sanitizeSceneName
...@@ -51,13 +49,13 @@ def shouldRefresh(exList): ...@@ -51,13 +49,13 @@ def shouldRefresh(exList):
""" """
MAX_REFRESH_AGE_SECS = 86400 # 1 day MAX_REFRESH_AGE_SECS = 86400 # 1 day
try: dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList) if not dbData:
lastRefresh = int(dbData['last_refreshed'])
return int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
except RecordNotFound:
return True return True
lastRefresh = int(dbData['last_refreshed'])
return int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
def setLastRefresh(exList): def setLastRefresh(exList):
""" """
...@@ -65,11 +63,11 @@ def setLastRefresh(exList): ...@@ -65,11 +63,11 @@ def setLastRefresh(exList):
:param exList: exception list to set refresh time :param exList: exception list to set refresh time
""" """
try: dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList) if dbData:
dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple())) dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple()))
sickrage.app.cache_db.update(dbData) sickrage.app.cache_db.update(dbData)
except RecordNotFound: else:
sickrage.app.cache_db.insert({ sickrage.app.cache_db.insert({
'_t': 'scene_exceptions_refresh', '_t': 'scene_exceptions_refresh',
'last_refreshed': int(time.mktime(datetime.datetime.today().timetuple())), 'last_refreshed': int(time.mktime(datetime.datetime.today().timetuple())),
...@@ -122,12 +120,13 @@ def retrieve_exceptions(get_xem=True, get_anidb=True): ...@@ -122,12 +120,13 @@ def retrieve_exceptions(get_xem=True, get_anidb=True):
_anidb_exceptions_fetcher() _anidb_exceptions_fetcher()
for cur_indexer_id, cur_exception_dict in exception_dict.items(): for cur_indexer_id, cur_exception_dict in exception_dict.items():
if not len(cur_exception_dict): continue if not len(cur_exception_dict):
continue
try: existing_exceptions = [x["show_name"] for x in
existing_exceptions = [x["show_name"] for x in sickrage.app.cache_db.get_many('scene_exceptions', cur_indexer_id)]
sickrage.app.cache_db.get_many('scene_exceptions', cur_indexer_id)]
except RecordNotFound: if not len(existing_exceptions):
continue continue
for cur_exception, curSeason in dict([(key, d[key]) for d in cur_exception_dict for key in d]).items(): for cur_exception, curSeason in dict([(key, d[key]) for d in cur_exception_dict for key in d]).items():
......
...@@ -23,8 +23,6 @@ import datetime ...@@ -23,8 +23,6 @@ import datetime
import time import time
import traceback import traceback
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from sickrage.core.helpers import findCertainShow, try_int from sickrage.core.helpers import findCertainShow, try_int
from sickrage.core.websession import WebSession from sickrage.core.websession import WebSession
...@@ -478,23 +476,23 @@ def xem_refresh(indexer_id, indexer, force=False): ...@@ -478,23 +476,23 @@ def xem_refresh(indexer_id, indexer, force=False):
MAX_REFRESH_AGE_SECS = 86400 # 1 day MAX_REFRESH_AGE_SECS = 86400 # 1 day
try: refresh = True
dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
if dbData:
lastRefresh = try_int(dbData['last_refreshed']) lastRefresh = try_int(dbData['last_refreshed'])
refresh = int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS refresh = int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
except RecordNotFound:
refresh = True
if refresh or force: if refresh or force:
sickrage.app.log.debug( sickrage.app.log.debug(
'Looking up XEM scene mapping for show %s on %s' % (indexer_id, IndexerApi(indexer).name)) 'Looking up XEM scene mapping for show %s on %s' % (indexer_id, IndexerApi(indexer).name))
# mark refreshed # mark refreshed
try: dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
dbData = sickrage.app.main_db.get('xem_refresh', indexer_id) if dbData:
dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple())) dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple()))
sickrage.app.main_db.update(dbData) sickrage.app.main_db.update(dbData)
except RecordNotFound: else:
sickrage.app.main_db.insert({ sickrage.app.main_db.insert({
'_t': 'xem_refresh', '_t': 'xem_refresh',
'indexer': indexer, 'indexer': indexer,
......
...@@ -25,8 +25,6 @@ import threading ...@@ -25,8 +25,6 @@ import threading
from collections import OrderedDict from collections import OrderedDict
from xml.etree.ElementTree import ElementTree from xml.etree.ElementTree import ElementTree
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from sickrage.core.common import Quality, UNKNOWN, UNAIRED, statusStrings, dateTimeFormat, SKIPPED, NAMING_EXTEND, \ from sickrage.core.common import Quality, UNKNOWN, UNAIRED, statusStrings, dateTimeFormat, SKIPPED, NAMING_EXTEND, \
NAMING_LIMITED_EXTEND, NAMING_LIMITED_EXTEND_E_PREFIXED, NAMING_DUPLICATE, NAMING_SEPARATED_REPEAT NAMING_LIMITED_EXTEND, NAMING_LIMITED_EXTEND_E_PREFIXED, NAMING_DUPLICATE, NAMING_SEPARATED_REPEAT
...@@ -773,14 +771,12 @@ class TVEpisode(object): ...@@ -773,14 +771,12 @@ class TVEpisode(object):
"release_group": self.release_group "release_group": self.release_group
} }
try: for x in sickrage.app.main_db.get_many('tv_episodes', self.show.indexerid):
for x in sickrage.app.main_db.get_many('tv_episodes', self.show.indexerid): if x['indexerid'] == self.indexerid:
if x['indexerid'] == self.indexerid: x.update(tv_episode)
x.update(tv_episode) sickrage.app.main_db.update(x)
sickrage.app.main_db.update(x) break
return else:
raise RecordNotFound
except RecordNotFound:
sickrage.app.main_db.insert(tv_episode) sickrage.app.main_db.insert(tv_episode)
def fullPath(self): def fullPath(self):
......
...@@ -28,7 +28,7 @@ import threading ...@@ -28,7 +28,7 @@ import threading
import traceback import traceback
import send2trash import send2trash
from CodernityDB.database import RecordNotFound, RevConflict from CodernityDB.database import RevConflict
from unidecode import unidecode from unidecode import unidecode
import sickrage import sickrage
...@@ -948,13 +948,10 @@ class TVShow(object): ...@@ -948,13 +948,10 @@ class TVShow(object):
self.release_groups = BlackAndWhiteList(self.indexerid) self.release_groups = BlackAndWhiteList(self.indexerid)
if not skipNFO: if not skipNFO:
try: # Get IMDb_info from database
# Get IMDb_info from database self._imdb_info = sickrage.app.main_db.get('imdb_info', self.indexerid)
self._imdb_info = sickrage.app.main_db.get('imdb_info', self.indexerid)
except RecordNotFound:
pass
def loadFromIndexer(self, cache=True, tvapi=None, cachedSeason=None): def loadFromIndexer(self, cache=True, tvapi=None):
if self.indexer is not INDEXER_TVRAGE: if self.indexer is not INDEXER_TVRAGE:
sickrage.app.log.debug( sickrage.app.log.debug(
...@@ -1039,16 +1036,17 @@ class TVShow(object): ...@@ -1039,16 +1036,17 @@ class TVShow(object):
try: try:
dbData = sickrage.app.main_db.get('imdb_info', self.indexerid) dbData = sickrage.app.main_db.get('imdb_info', self.indexerid)
dbData.update(self.imdb_info) if dbData:
sickrage.app.main_db.update(dbData) dbData.update(self.imdb_info)
sickrage.app.main_db.update(dbData)
else:
imdb_info.update(self.imdb_info)
sickrage.app.main_db.insert(imdb_info)
except RevConflict: except RevConflict:
dbData = sickrage.app.main_db.get('imdb_info', self.indexerid) dbData = sickrage.app.main_db.get('imdb_info', self.indexerid)
sickrage.app.main_db.delete(dbData) sickrage.app.main_db.delete(dbData)
imdb_info.update(self.imdb_info) imdb_info.update(self.imdb_info)
sickrage.app.main_db.insert(imdb_info) sickrage.app.main_db.insert(imdb_info)
except RecordNotFound:
imdb_info.update(self.imdb_info)
sickrage.app.main_db.insert(imdb_info)
def deleteShow(self, full=False): def deleteShow(self, full=False):
# choose delete or trash action # choose delete or trash action
...@@ -1256,11 +1254,11 @@ class TVShow(object): ...@@ -1256,11 +1254,11 @@ class TVShow(object):
"search_delay": self.search_delay, "search_delay": self.search_delay,
} }
try: dbData = sickrage.app.main_db.get('tv_shows', self.indexerid)
dbData = sickrage.app.main_db.get('tv_shows', self.indexerid) if dbData:
dbData.update(tv_show) dbData.update(tv_show)
sickrage.app.main_db.update(dbData) sickrage.app.main_db.update(dbData)
except RecordNotFound: else:
sickrage.app.main_db.insert(tv_show) sickrage.app.main_db.insert(tv_show)
def __str__(self): def __str__(self):
......
...@@ -22,8 +22,6 @@ import datetime ...@@ -22,8 +22,6 @@ import datetime
import threading import threading
import time import time
from CodernityDB.database import RecordNotFound
import sickrage import sickrage
from sickrage.core.exceptions import CantRefreshShowException, CantUpdateShowException from sickrage.core.exceptions import CantRefreshShowException, CantUpdateShowException
from sickrage.core.ui import ProgressIndicators, QueueProgressIndicator from sickrage.core.ui import ProgressIndicators, QueueProgressIndicator
...@@ -47,10 +45,10 @@ class ShowUpdater(object): ...@@ -47,10 +45,10 @@ class ShowUpdater(object):
update_timestamp = int(time.mktime(datetime.datetime.now().timetuple())) update_timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
try: dbData = sickrage.app.cache_db.get('lastUpdate', 'theTVDB')
dbData = sickrage.app.cache_db.get('lastUpdate', 'theTVDB') if dbData:
last_update = int(dbData['time']) last_update = int(dbData['time'])
except RecordNotFound: else:
last_update = update_timestamp last_update = update_timestamp
dbData = sickrage.app.cache_db.insert({ dbData = sickrage.app.cache_db.insert({
'_t': 'lastUpdate', '_t': 'lastUpdate',
......
...@@ -23,7 +23,6 @@ import re ...@@ -23,7 +23,6 @@ import re
import threading import threading
from datetime import datetime from datetime import datetime
from CodernityDB.database import RecordNotFound
from dateutil import tz from dateutil import tz
import sickrage import sickrage
...@@ -71,30 +70,23 @@ class TimeZoneUpdater(object): ...@@ -71,30 +70,23 @@ class TimeZoneUpdater(object):
for network, timezone in d.items(): for network, timezone in d.items():
existing =