bumped cryptography dependency

modified renovate.json
made sqlite the default database type
refactored unit tests
added params to silence logging during shutdown
added test stage to CI using pytest and tox
updated .gitignore
fixed issues with quiet console logging
removed all messages.json files

.gitignore

@@ -4,10 +4,6 @@
 /sickrage/libs/adba/anime-list.xml
 /sickrage/libs/adba/animetitles.xml
 
-# SR Tox Files #
-######################
-/.tox/
-
 # SR GitLab Files #
 ######################
 /.gitlab/
@@ -20,9 +16,9 @@
 # SR User Related #
 ######################
 *.db*
-*.ini
 *.torrent
 *.magnet
+config.ini
 swagger.json
 privatekey.pem
 autoProcessTV.cfg
@@ -33,11 +29,9 @@ autoProcessTV.cfg
 
 # SR Test Related #
 ######################
-/tests/database/
-/tests/providers/
-/tests/show name/
-/tests/show name final/
-/tests/sr_update_*/
+/tests/data/
+/.tox/
+report.xml
 
 # Compiled Source #
 ######################

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
  # - review_webpack
  # - review_docker
  # - review_deploy
+ - test
  - build
  - sentry
  - deploy
@@ -99,27 +100,61 @@ stages:
 # only:
 # - [email protected]/sickrage
 
-build:
- stage: build
- retry: 2
+.test_template: &test
+ stage: test
+ retry: 1
  image:
- name: nikolaik/python-nodejs:python3.9-nodejs10-alpine
+ name: python:$PYTHON_VERSION
  variables:
- NODE_ENV: "development"
- CARGO_HOME: "$CI_PROJECT_DIR/cargo"
+ ASYNC_TEST_TIMEOUT: 60
  script:
- - export PATH="$CARGO_HOME/bin:$PATH"
- - apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- - curl https://sh.rustup.rs -sSf | sh -s -- -y
- - export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache
- - pip install -U pip
- - pip install -r requirements-dev.txt
- - yarn run build
- - python checksum-generator.py
- only:
- - branches
+ - pip install tox
+ - tox -e $TOX_ENV
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+ paths:
+ - report.xml
+ expire_in: 1 week
  except:
- - [email protected]/sickrage
+ refs:
+ - tags
+ - triggers
+ variables:
+ - $CI_COMMIT_BRANCH == "master"
+ - $CI_COMMIT_MESSAGE =~ /\[TASK\] Pre-Releasing/
+ - $CI_COMMIT_MESSAGE =~ /\[TASK\] Bump/
+
+test_py36:
+ <<: *test
+ variables:
+ TOX_ENV: "py36"
+ PYTHON_VERSION: "3.6"
+
+test_py37:
+ <<: *test
+ variables:
+ TOX_ENV: "py37"
+ PYTHON_VERSION: "3.7"
+
+test_py38:
+ <<: *test
+ variables:
+ TOX_ENV: "py38"
+ PYTHON_VERSION: "3.8"
+
+test_py39:
+ <<: *test
+ variables:
+ TOX_ENV: "py39"
+ PYTHON_VERSION: "3.9"
+
+test_py310:
+ <<: *test
+ variables:
+ TOX_ENV: "py310"
+ PYTHON_VERSION: "3.10"
 
 build_master:
  stage: build
@@ -265,18 +300,6 @@ pypi:
  - branches
  - triggers
 
-docker:
- stage: deploy
- script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
- only:
- - branches
- except:
- - [email protected]/sickrage
-
 docker_master:
  stage: deploy
  trigger:

renovate.json

 {
  "extends": [
- "config:base"
+ "config:base",
+ ":assignee(echel0n)"
  ],
- "prHourlyLimit": 2,
- "rebaseWhen": "conflicted",
  "baseBranches": [
  "develop"
  ],
+ "enabledManagers": [
+ "pip_requirements",
+ "poetry"
+ ],
  "python": {
  "commitMessageAction": "Update Python",
  "managerBranchPrefix": "py/",
- "labels": ["Update dep (Py)"]
+ "labels": [
+ "dependencies",
+ "python"
+ ]
  }
-}
+}
\ No newline at end of file

requirements-dev.txt

@@ -2,5 +2,5 @@ twine
 crowdin-cli-py
 babel
 tox
-vcrpy-unittest
 mako
+pytest
\ No newline at end of file

requirements.txt

@@ -19,7 +19,7 @@ cleverdict==1.9.2
 click==7.1.2
 cloudscraper==1.2.46
 configobj==5.0.6
-cryptography==3.2.1
+cryptography==3.3.2
 decorator==4.4.2
 deluge-client==1.9.0
 dirsync==2.2.5

setup.cfg

@@ -40,3 +40,19 @@ output_dir = sickrage/locale
 input_file = sickrage/locale/messages.pot
 ignore_obsolete = true
 previous = true
+
+[tool:pytest]
+python_files = tests/*
+filterwarnings =
+ error
+ ignore::sqlalchemy.exc.SAWarning
+ ignore::pytest.PytestUnraisableExceptionWarning
+ ignore:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning
+ ignore:There is no current event loop:DeprecationWarning
+ ignore:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning
+ ignore:ssl\.match_hostname\(\) is deprecated:DeprecationWarning
+ ignore:the load_module\(\) method is deprecated.*:DeprecationWarning
+ ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning
+ ignore:EntryPoint tuple interface is deprecated. Access members by name:DeprecationWarning
+ ignore:_SixMetaPathImporter\.exec_module\(\) not found; falling back to load_module\(\):ImportWarning
+ ignore:_SixMetaPathImporter\.find_spec\(\) not found; falling back to find_module\(\):ImportWarning

setup.py

@@ -8,13 +8,8 @@ from sickrage import __version__
 
 
 def requirements():
- with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements.txt'))) as f:
- return f.read().splitlines()
-
-
-def requirements_dev():
- with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements-dev.txt'))) as f:
- return f.read().splitlines()
+ with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'requirements.txt')) as f:
+ return f.read().splitlines(keepends=False)
 
 
 class CleanCommand(Command):
@@ -61,9 +56,6 @@ setup(
  keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'],
  packages=['sickrage'],
  install_requires=requirements(),
- extras_require={
- 'dev': requirements_dev()
- },
  include_package_data=True,
  python_requires='>=3',
  platforms='any',

sickrage/__init__.py

@@ -34,13 +34,10 @@ import gettext
 import multiprocessing
 import os
 import pathlib
-import re
 import site
-import subprocess
 import threading
 import time
 import traceback
-import pkg_resources
 
 # pywin32 for windows service
 try:
@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False):
  sys.path.extend(remainder)
 
 # set system default language
-gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
+gettext.install('messages', LOCALE_DIR, names=["ngettext"])
 
 if __install_type__ == 'windows':
  class SiCKRAGEService(win32serviceutil.ServiceFramework):
@@ -248,31 +245,6 @@ def changelog():
  return f.read()
 
 
-def check_requirements():
- if os.path.exists(REQS_FILE):
- with open(REQS_FILE) as f:
- for line in f.readlines():
- try:
- req_name, req_version = line.strip().split('==', 1)
- if 'python_version' in req_version:
- m = re.search('(\d+.\d+.\d+).*(\d+.\d+)', req_version)
- req_version = m.group(1)
- python_version = m.group(2)
- python_version_major = int(python_version.split('.')[0])
- python_version_minor = int(python_version.split('.')[1])
- if sys.version_info.major == python_version_major and sys.version_info.minor != python_version_minor:
- continue
-
- if not pkg_resources.get_distribution(req_name).version == req_version:
- print('Updating requirement {} to {}'.format(req_name, req_version))
- subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
- except pkg_resources.DistributionNotFound:
- print('Installing requirement {}'.format(line.strip()))
- subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
- except ValueError:
- continue
-
-
 def verify_checksums(remove_unverified=False):
  valid_files = []
  exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)]
@@ -484,6 +456,9 @@ def start():
  app.pid = app.daemon.pid
 
  app.start()
+
+ from tornado.ioloop import IOLoop
+ IOLoop.current().start()
  except (SystemExit, KeyboardInterrupt):
  if app:
  app.shutdown()

sickrage/core/__init__.py

@@ -256,7 +256,7 @@ class Core(object):
  self.metadata_providers = MetadataProviders()
  self.search_providers = SearchProviders()
  self.series_providers = SeriesProviders()
- self.log = Logger()
+ self.log = Logger(consoleLogging=not self.quiet, logFile=os.path.join(self.data_dir, 'logs', 'sickrage.log'))
  self.alerts = Notifications()
  self.wserver = WebServer()
  self.show_queue = ShowQueue()
@@ -347,9 +347,7 @@ class Core(object):
  # setup logger settings
  self.log.logSize = self.config.general.log_size
  self.log.logNr = self.config.general.log_nr
- self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
  self.log.debugLogging = self.debug or self.config.general.debug
- self.log.consoleLogging = not self.quiet
 
  # start logger
  self.log.start()
@@ -570,9 +568,6 @@ class Core(object):
  # perform shutdown trigger check every 5 seconds
  PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
 
- # start ioloop
- IOLoop.current().start()
-
  def init_sentry(self):
  # sentry log handler
  sentry_logging = LoggingIntegration(
@@ -732,6 +727,11 @@ class Core(object):
  # save settings
  self.config.save()
 
+ # shutdown databases
+ self.main_db.shutdown()
+ self.config.db.shutdown()
+ self.cache_db.shutdown()
+
  # shutdown logging
  if self.log:
  self.log.close()

sickrage/core/api/__init__.py

@@ -110,7 +110,7 @@ class API(object):
  def health(self):
  for i in range(3):
  try:
- health = requests.get(urljoin(self.api_base, "health"), verify=False, timeout=30).ok
+ health = requests.get(urljoin(self.api_base, "health"), verify=True, timeout=30).ok
  except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
  pass
  else:

sickrage/core/config/helpers.py

@@ -110,7 +110,7 @@ def change_gui_lang(language):
  gt.install(names=["ngettext"])
  else:
  # System default language
- gettext.install('messages', sickrage.LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
+ gettext.install('messages', sickrage.LOCALE_DIR, names=["ngettext"])
 
  sickrage.app.config.gui.gui_lang = language
 

sickrage/core/databases/__init__.py

@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs):
 
 class IntFlag(TypeDecorator):
  impl = sqlalchemy.types.Integer()
+ cache_ok = True
 
  def __init__(self, enum):
  self.enum = enum
@@ -152,20 +153,21 @@ class SRDatabase(object):
 
  @property
  def engine(self):
- if self.db_type == 'sqlite':
- return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
- elif self.db_type == 'mysql':
+ if self.db_type == 'mysql':
  mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False)
  mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}")
  return create_engine(
  'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name),
  echo=False)
+ else:
+ return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
 
  @property
  def version(self):
- context = MigrationContext.configure(self.engine)
- current_rev = context.get_current_revision()
- return current_rev
+ with self.engine.connect() as connection:
+ context = MigrationContext.configure(connection)
+ current_rev = context.get_current_revision()
+ return current_rev
 
  def setup(self):
  if inspect(self.engine).has_table('migrate_version'):
@@ -321,3 +323,6 @@ class SRDatabase(object):
  rows.append(row._asdict())
  session.bulk_insert_mappings(table, rows)
  session.commit()
+
+ def shutdown(self):
+ self.session.close()

sickrage/core/helpers/show_names.py

@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words):
  """
  if isinstance(words, str):
  words = words.split(',')
- items = [(re.compile('(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
+ items = [(re.compile(r'(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
  for regexp, word in items:
  if regexp.search(name):
  return word

sickrage/core/imdb_popular.py

@@ -95,7 +95,7 @@ class imdbPopular(object):
 
  @staticmethod
  def change_size(image_url, factor=3):
- match = re.search("^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
+ match = re.search(r"^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
 
  if match:
  matches = match.groups()

sickrage/core/logger/__init__.py

@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()):
  if self.logFile:
  # make logs folder if it doesn't exist
  if not os.path.exists(os.path.dirname(self.logFile)):
- if not make_dir(os.path.dirname(self.logFile)):
+ if not os.makedirs(os.path.dirname(self.logFile)):
  return
 
  if sickrage.app.developer:

sickrage/core/nzbSplitter.py

@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season):
 
  filename = name.replace(".nzb", "")
 
- regex = '([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
+ regex = r'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
 
  sceneNameMatch = re.search(regex, filename, re.I)
  if sceneNameMatch:
@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season):
  sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.")
  return {}, ''
 
- regex = '(' + re.escape(showName) + '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
+ regex = '(' + re.escape(showName) + r'\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
  regex = regex.replace(' ', '.')
 
  epFiles = {}
  xmlns = None
 
  for curFile in nzbElement.getchildren():
- xmlnsMatch = re.match("{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
+ xmlnsMatch = re.match(r"{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
  if not xmlnsMatch:
  continue
  else:

sickrage/core/queues/__init__.py

@@ -84,7 +84,7 @@ class Queue(object):
  ids.append(worker_id)
 
  self.auto_remove_tasks_timer = threading.Timer(10.0, self.auto_remove_tasks)
- self.auto_remove_tasks_timer.setName(self.name)
+ self.auto_remove_tasks_timer.name = self.name
  self.auto_remove_tasks_timer.start()
 
  return ids
@@ -94,12 +94,14 @@ class Queue(object):
  This function stops and kills a worker.
  If no id is provided, all workers are killed.
  :param worker_id: the identifier for the worker to kill
+ :param quiet: silence logging
  :return: None
  """
  try:
  self.lock.acquire()
  if worker_id is None:
  sickrage.app.log.info("Shutting down all {} workers".format(self.name))
+
  for worker in self.workers:
  worker.must_die = True
  else:

sickrage/core/webserver/handlers/api/__init__.py

@@ -50,11 +50,11 @@ class APIBaseHandler(RequestHandler):
 
  method_name = self.request.method.lower()
  if method_name == 'options':
- return
+ return self.finish()
 
  certs = sickrage.app.auth_server.certs()
  if not certs:
- return
+ return self.finish()
 
  auth_header = self.request.headers.get('Authorization')
 
@@ -93,7 +93,7 @@ class APIBaseHandler(RequestHandler):
  })
 
  if sickrage.app.config.user.sub_id != decoded_token.get('sub'):
- return self._unauthorized(error='user is not authorized')
+ return self.finish(self._unauthorized(error='user is not authorized'))
 
  if not sickrage.app.config.general.sso_api_key:
  sickrage.app.config.general.sso_api_key = decoded_token.get('apikey')
@@ -117,11 +117,11 @@ class APIBaseHandler(RequestHandler):
  method = self.run_async(getattr(self, method_name))
  setattr(self, method_name, method)
  except Exception:
- return self._unauthorized(error='failed to decode token')
+ self.finish(self._unauthorized(error='failed to decode token'))
  else:
- return self._unauthorized(error='invalid authorization request')
+ self.finish(self._unauthorized(error='invalid authorization request'))
  else:
- return self._unauthorized(error='authorization header missing')
+ self.finish(self._unauthorized(error='authorization header missing'))
 
  def run_async(self, method):
  @functools.wraps(method)

sickrage/core/webserver/handlers/home/__init__.py

@@ -166,7 +166,7 @@ class IsAliveHandler(BaseHandler):
  if not srcallback:
  return _("Error: Unsupported Request. Send jsonp request with 'srcallback' variable in the query string.")
 
- return "{}({})".format(srcallback, {'msg': str(sickrage.app.pid) if sickrage.app.started else 'nope'})
+ return '{}({})'.format(srcallback, {"msg": str(sickrage.app.pid) if sickrage.app.started else 'nope'})
 
 
 class TestSABnzbdHandler(BaseHandler):