Refactored get, all, and get_many database functions.

changelog.md

 # Changelog
 
-- * c1ad198 - 2018-11-11: Release v9.4.29 
+- * 0e4c60f - 2018-11-11: Refactored get, all, and get_many database functions. 
+- * f87d6ad - 2018-11-11: Release v9.4.29 
 - * 7d0812a - 2018-11-11: Pre-Release v9.4.29.dev2 
 - * 8028ae9 - 2018-11-11: Default subtitle language of English choosen if no language is specified in subtitle settings 
 - * 5eef56e - 2018-11-11: Pre-Release v9.4.29.dev1 

sickrage/core/blackandwhitelist.py

@@ -19,8 +19,6 @@
 
 from __future__ import unicode_literals
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from adba.aniDBerrors import AniDBCommandTimeoutError
 
@@ -85,10 +83,8 @@ class BlackAndWhiteList(object):
 
  :param table: database table remove keywords from
  """
- try:
- sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
- except RecordNotFound:
- pass
+ sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
+
 
  def _load_list(self, table):
  """

sickrage/core/caches/name_cache.py

@@ -22,8 +22,6 @@ from __future__ import unicode_literals
 import time
 from datetime import datetime, timedelta
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from sickrage.core.helpers import full_sanitizeSceneName
 from sickrage.core.helpers.encoding import strip_accents
@@ -55,16 +53,8 @@ class NameCache(object):
 
  self.cache[name] = int(indexer_id)
 
- try:
- if not len([x for x in sickrage.app.cache_db.get_many('scene_names', name)
- if x['indexer_id'] == indexer_id]):
- # insert name into cache
- sickrage.app.cache_db.insert({
- '_t': 'scene_names',
- 'indexer_id': indexer_id,
- 'name': name
- })
- except RecordNotFound:
+ dbData = [x for x in sickrage.app.cache_db.get_many('scene_names', name) if x['indexer_id'] == indexer_id]
+ if not len(dbData):
  # insert name into cache
  sickrage.app.cache_db.insert({
  '_t': 'scene_names',
@@ -102,12 +92,9 @@ class NameCache(object):
  def save(self):
  """Commit cache to database file"""
  for name, indexer_id in self.cache.items():
- try:
- if len([x for x in sickrage.app.cache_db.get_many('scene_names', name)
- if x['indexer_id'] == indexer_id]):
- continue
- except RecordNotFound:
- pass
+ dbData = [x for x in sickrage.app.cache_db.get_many('scene_names', name) if x['indexer_id'] == indexer_id]
+ if len(dbData):
+ continue
 
  # insert name into cache
  sickrage.app.cache_db.insert({

sickrage/core/caches/tv_cache.py

@@ -22,8 +22,6 @@ import datetime
 import time
 
 import feedparser
-from CodernityDB.database import RecordNotFound
-from CodernityDB.index import IndexNotFoundException
 
 import sickrage
 from sickrage.core.api.cache import ProviderCacheAPI
@@ -121,23 +119,23 @@ class TVCache(object):
 
  @property
  def last_update(self):
- try:
-   dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
+ dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
+ if dbData:
  lastTime = int(dbData["time"])
  if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
  lastTime = 0
- except (RecordNotFound, IndexNotFoundException):
+ else:
  lastTime = 0
 
  return datetime.datetime.fromtimestamp(lastTime)
 
  @last_update.setter
  def last_update(self, toDate):
- try:
-   dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
+ dbData = sickrage.app.cache_db.get('lastUpdate', self.providerID)
+ if dbData:
  dbData['time'] = int(time.mktime(toDate.timetuple()))
  sickrage.app.cache_db.update(dbData)
- except (RecordNotFound, IndexNotFoundException):
+ else:
  sickrage.app.cache_db.insert({
  '_t': 'lastUpdate',
  'provider': self.providerID,
@@ -146,23 +144,23 @@ class TVCache(object):
 
  @property
  def last_search(self):
- try:
-   dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
+ dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
+ if dbData:
  lastTime = int(dbData["time"])
  if lastTime > int(time.mktime(datetime.datetime.today().timetuple())):
  lastTime = 0
- except (RecordNotFound, IndexNotFoundException):
+ else:
  lastTime = 0
 
  return datetime.datetime.fromtimestamp(lastTime)
 
  @last_search.setter
  def last_search(self, toDate):
- try:
-   dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
+ dbData = sickrage.app.cache_db.get('lastSearch', self.providerID)
+ if dbData:
  dbData['time'] = int(time.mktime(toDate.timetuple()))
  sickrage.app.cache_db.update(dbData)
- except (RecordNotFound, IndexNotFoundException):
+ else:
  sickrage.app.cache_db.insert({
  '_t': 'lastUpdate',
  'provider': self.providerID,

sickrage/core/databases/__init__.py

@@ -26,6 +26,7 @@ import time
 import traceback
 from sqlite3 import OperationalError
 
+from CodernityDB.database import RecordDeleted, RecordNotFound
 from CodernityDB.database_super_thread_safe import SuperThreadSafeDatabase
 from CodernityDB.index import IndexNotFoundException, IndexConflict, IndexException
 from CodernityDB.storage import IU_Storage
@@ -334,15 +335,46 @@ class srDatabase(object):
  if os.path.isfile(self.old_db_path + '-shm'):
  os.rename(self.old_db_path + '-shm', '{}-shm.{}_old'.format(self.old_db_path, random))
 
+ def delete_corrupted(self, _id, traceback_error=''):
+ try:
+ sickrage.app.log.debug('Deleted corrupted document "{}": {}'.format(_id, traceback_error))
+ corrupted = self.db.get('id', _id, with_storage=False)
+ self.db._delete_id_index(corrupted.get('_id'), corrupted.get('_rev'), None)
+ except:
+ log.debug('Failed deleting corrupted: {}'.format(traceback.format_exc()))
+
  def all(self, *args, **kwargs):
- return (x['doc'] for x in self.db.all(with_doc=True, *args, **kwargs))
+ for data in self.db.all(*args, **kwargs):
+ if kwargs.pop('with_doc', True):
+ try:
+ doc = self.db.get('id', data['_id'])
+ yield doc
+ except (RecordDeleted, RecordNotFound):
+ sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
+ except (ValueError, EOFError):
+ self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
+ else:
+ yield data
 
  def get_many(self, *args, **kwargs):
- return (x['doc'] for x in self.db.get_many(with_doc=True, *args, **kwargs))
+ for data in self.db.get_many(*args, **kwargs):
+ if kwargs.pop('with_doc', True):
+ try:
+ doc = self.db.get('id', data['_id'])
+ yield doc
+ except (RecordDeleted, RecordNotFound):
+ sickrage.app.log.debug('Record not found, skipping: {}'.format(data['_id']))
+ except (ValueError, EOFError):
+ self.delete_corrupted(data.get('_id'), traceback_error=traceback.format_exc(0))
+ else:
+ yield data
 
  def get(self, *args, **kwargs):
- x = self.db.get(with_doc=True, *args, **kwargs)
- return x.get('doc', x)
+ try:
+ x = self.db.get(with_doc=kwargs.pop('with_doc', True), *args, **kwargs)
+ return x.get('doc', x)
+ except (RecordDeleted, RecordNotFound):
+ pass
 
  def delete(self, *args):
  return self.db.delete(*args)

sickrage/core/scene_exceptions.py

@@ -24,8 +24,6 @@ import re
 import threading
 import time
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from adba.aniDBAbstracter import Anime
 from sickrage.core.helpers import full_sanitizeSceneName, sanitizeSceneName
@@ -51,13 +49,13 @@ def shouldRefresh(exList):
  """
  MAX_REFRESH_AGE_SECS = 86400 # 1 day
 
- try:
- dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
- lastRefresh = int(dbData['last_refreshed'])
- return int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
- except RecordNotFound:
+ dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
+ if not dbData:
  return True
 
+ lastRefresh = int(dbData['last_refreshed'])
+ return int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
+
 
 def setLastRefresh(exList):
  """
@@ -65,11 +63,11 @@ def setLastRefresh(exList):
 
  :param exList: exception list to set refresh time
  """
- try:
-   dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
+ dbData = sickrage.app.cache_db.get('scene_exceptions_refresh', exList)
+ if dbData:
  dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple()))
  sickrage.app.cache_db.update(dbData)
- except RecordNotFound:
+ else:
  sickrage.app.cache_db.insert({
  '_t': 'scene_exceptions_refresh',
  'last_refreshed': int(time.mktime(datetime.datetime.today().timetuple())),
@@ -122,12 +120,13 @@ def retrieve_exceptions(get_xem=True, get_anidb=True):
  _anidb_exceptions_fetcher()
 
  for cur_indexer_id, cur_exception_dict in exception_dict.items():
- if not len(cur_exception_dict): continue
+ if not len(cur_exception_dict):
+ continue
 
- try:
- existing_exceptions = [x["show_name"] for x in
- sickrage.app.cache_db.get_many('scene_exceptions', cur_indexer_id)]
- except RecordNotFound:
+ existing_exceptions = [x["show_name"] for x in
+  sickrage.app.cache_db.get_many('scene_exceptions', cur_indexer_id)]
+
+ if not len(existing_exceptions):
  continue
 
  for cur_exception, curSeason in dict([(key, d[key]) for d in cur_exception_dict for key in d]).items():

sickrage/core/scene_numbering.py

@@ -23,8 +23,6 @@ import datetime
 import time
 import traceback
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from sickrage.core.helpers import findCertainShow, try_int
 from sickrage.core.websession import WebSession
@@ -478,23 +476,23 @@ def xem_refresh(indexer_id, indexer, force=False):
 
  MAX_REFRESH_AGE_SECS = 86400 # 1 day
 
- try:
- dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
+ refresh = True
+
+ dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
+ if dbData:
  lastRefresh = try_int(dbData['last_refreshed'])
  refresh = int(time.mktime(datetime.datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
- except RecordNotFound:
- refresh = True
 
  if refresh or force:
  sickrage.app.log.debug(
  'Looking up XEM scene mapping for show %s on %s' % (indexer_id, IndexerApi(indexer).name))
 
  # mark refreshed
- try:
-   dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
+ dbData = sickrage.app.main_db.get('xem_refresh', indexer_id)
+ if dbData:
  dbData['last_refreshed'] = int(time.mktime(datetime.datetime.today().timetuple()))
  sickrage.app.main_db.update(dbData)
- except RecordNotFound:
+ else:
  sickrage.app.main_db.insert({
  '_t': 'xem_refresh',
  'indexer': indexer,

sickrage/core/tv/episode/__init__.py

@@ -25,8 +25,6 @@ import threading
 from collections import OrderedDict
 from xml.etree.ElementTree import ElementTree
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from sickrage.core.common import Quality, UNKNOWN, UNAIRED, statusStrings, dateTimeFormat, SKIPPED, NAMING_EXTEND, \
  NAMING_LIMITED_EXTEND, NAMING_LIMITED_EXTEND_E_PREFIXED, NAMING_DUPLICATE, NAMING_SEPARATED_REPEAT
@@ -773,14 +771,12 @@ class TVEpisode(object):
  "release_group": self.release_group
  }
 
- try:
- for x in sickrage.app.main_db.get_many('tv_episodes', self.show.indexerid):
- if x['indexerid'] == self.indexerid:
- x.update(tv_episode)
- sickrage.app.main_db.update(x)
- return
- raise RecordNotFound
- except RecordNotFound:
+ for x in sickrage.app.main_db.get_many('tv_episodes', self.show.indexerid):
+ if x['indexerid'] == self.indexerid:
+ x.update(tv_episode)
+ sickrage.app.main_db.update(x)
+ break
+ else:
  sickrage.app.main_db.insert(tv_episode)
 
  def fullPath(self):

sickrage/core/tv/show/__init__.py

@@ -28,7 +28,7 @@ import threading
 import traceback
 
 import send2trash
-from CodernityDB.database import RecordNotFound, RevConflict
+from CodernityDB.database import RevConflict
 from unidecode import unidecode
 
 import sickrage
@@ -948,13 +948,10 @@ class TVShow(object):
  self.release_groups = BlackAndWhiteList(self.indexerid)
 
  if not skipNFO:
- try:
- # Get IMDb_info from database
- self._imdb_info = sickrage.app.main_db.get('imdb_info', self.indexerid)
- except RecordNotFound:
- pass
+ # Get IMDb_info from database
+ self._imdb_info = sickrage.app.main_db.get('imdb_info', self.indexerid)
 
- def loadFromIndexer(self, cache=True, tvapi=None, cachedSeason=None):
+ def loadFromIndexer(self, cache=True, tvapi=None):
 
  if self.indexer is not INDEXER_TVRAGE:
  sickrage.app.log.debug(
@@ -1039,16 +1036,17 @@ class TVShow(object):
 
  try:
  dbData = sickrage.app.main_db.get('imdb_info', self.indexerid)
- dbData.update(self.imdb_info)
- sickrage.app.main_db.update(dbData)
+ if dbData:
+ dbData.update(self.imdb_info)
+ sickrage.app.main_db.update(dbData)
+ else:
+ imdb_info.update(self.imdb_info)
+ sickrage.app.main_db.insert(imdb_info)
  except RevConflict:
  dbData = sickrage.app.main_db.get('imdb_info', self.indexerid)
  sickrage.app.main_db.delete(dbData)
  imdb_info.update(self.imdb_info)
  sickrage.app.main_db.insert(imdb_info)
- except RecordNotFound:
- imdb_info.update(self.imdb_info)
- sickrage.app.main_db.insert(imdb_info)
 
  def deleteShow(self, full=False):
  # choose delete or trash action
@@ -1256,11 +1254,11 @@ class TVShow(object):
  "search_delay": self.search_delay,
  }
 
- try:
-   dbData = sickrage.app.main_db.get('tv_shows', self.indexerid)
+ dbData = sickrage.app.main_db.get('tv_shows', self.indexerid)
+ if dbData:
  dbData.update(tv_show)
  sickrage.app.main_db.update(dbData)
- except RecordNotFound:
+ else:
  sickrage.app.main_db.insert(tv_show)
 
  def __str__(self):

sickrage/core/updaters/show_updater.py

@@ -22,8 +22,6 @@ import datetime
 import threading
 import time
 
-from CodernityDB.database import RecordNotFound
-
 import sickrage
 from sickrage.core.exceptions import CantRefreshShowException, CantUpdateShowException
 from sickrage.core.ui import ProgressIndicators, QueueProgressIndicator
@@ -47,10 +45,10 @@ class ShowUpdater(object):
 
  update_timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
 
- try:
-   dbData = sickrage.app.cache_db.get('lastUpdate', 'theTVDB')
+ dbData = sickrage.app.cache_db.get('lastUpdate', 'theTVDB')
+ if dbData:
  last_update = int(dbData['time'])
- except RecordNotFound:
+ else:
  last_update = update_timestamp
  dbData = sickrage.app.cache_db.insert({
  '_t': 'lastUpdate',

sickrage/core/updaters/tz_updater.py

@@ -23,7 +23,6 @@ import re
 import threading
 from datetime import datetime
 
-from CodernityDB.database import RecordNotFound
 from dateutil import tz
 
 import sickrage
@@ -71,30 +70,23 @@ class TimeZoneUpdater(object):
  for network, timezone in d.items():
  existing = network in self.network_dict
  if not existing:
- try:
- sickrage.app.cache_db.get('network_timezones', network)
- except RecordNotFound:
+ if not sickrage.app.cache_db.get('network_timezones', network):
  sickrage.app.cache_db.insert({
  '_t': 'network_timezones',
  'network_name': ss(network),
  'timezone': timezone
  })
  elif self.network_dict[network] is not timezone:
- try:
-   dbData = sickrage.app.cache_db.get('network_timezones', network)
+ dbData = sickrage.app.cache_db.get('network_timezones', network)
+ if dbData:
  dbData['timezone'] = timezone
  sickrage.app.cache_db.update(dbData)
- except RecordNotFound:
- continue
 
  if existing:
  del self.network_dict[network]
 
  for x in self.network_dict:
- try:
- sickrage.app.cache_db.delete(sickrage.app.cache_db.get('network_timezones', x))
- except RecordNotFound:
- continue
+ sickrage.app.cache_db.delete(sickrage.app.cache_db.get('network_timezones', x))
 
  self.load_network_dict()
 

sickrage/core/webserver/views.py

@@ -32,7 +32,6 @@ from urlparse import urlparse, urljoin
 import dateutil.tz
 import markdown2
 import tornado.locale
-from CodernityDB.database import RecordNotFound
 from concurrent.futures import ThreadPoolExecutor
 from mako.exceptions import RichTraceback
 from mako.lookup import TemplateLookup
@@ -3475,10 +3474,7 @@ class Manage(Home, WebRoot):
  toRemove = toRemove.split("|") if toRemove is not None else []
 
  for release in toRemove:
- try:
- [sickrage.app.main_db.delete(x) for x in sickrage.app.main_db.get_many('failed_snatches', release)]
- except RecordNotFound:
- continue
+ [sickrage.app.main_db.delete(x) for x in sickrage.app.main_db.get_many('failed_snatches', release)]
 
  if toRemove:
  return self.redirect('/manage/failedDownloads/')