Merge branch 'develop'

# Conflicts:
#	renovate.json

.gitignore

@@ -4,10 +4,6 @@
 /sickrage/libs/adba/anime-list.xml
 /sickrage/libs/adba/animetitles.xml
 
-# SR Tox Files #
-######################
-/.tox/
-
 # SR GitLab Files #
 ######################
 /.gitlab/
@@ -20,9 +16,9 @@
 # SR User Related #
 ######################
 *.db*
-*.ini
 *.torrent
 *.magnet
+config.ini
 swagger.json
 privatekey.pem
 autoProcessTV.cfg
@@ -33,11 +29,9 @@ autoProcessTV.cfg
 
 # SR Test Related #
 ######################
-/tests/database/
-/tests/providers/
-/tests/show name/
-/tests/show name final/
-/tests/sr_update_*/
+/tests/data/
+/.tox/
+report.xml
 
 # Compiled Source #
 ######################

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
  # - review_webpack
  # - review_docker
  # - review_deploy
+ - test
  - build
  - sentry
  - deploy
@@ -99,27 +100,61 @@ stages:
 # only:
 # - [email protected]/sickrage
 
-build:
- stage: build
- retry: 2
+.test_template: &test
+ stage: test
+ retry: 1
  image:
- name: nikolaik/python-nodejs:python3.9-nodejs10-alpine
+ name: python:$PYTHON_VERSION
  variables:
- NODE_ENV: "development"
- CARGO_HOME: "$CI_PROJECT_DIR/cargo"
+ ASYNC_TEST_TIMEOUT: 60
  script:
- - export PATH="$CARGO_HOME/bin:$PATH"
- - apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- - curl https://sh.rustup.rs -sSf | sh -s -- -y
- - export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache
- - pip install -U pip
- - pip install -r requirements-dev.txt
- - yarn run build
- - python checksum-generator.py
- only:
- - branches
+ - pip install tox
+ - tox -e $TOX_ENV
+ artifacts:
+ when: always
+ reports:
+ junit: report.xml
+ paths:
+ - report.xml
+ expire_in: 1 week
  except:
- - [email protected]/sickrage
+ refs:
+ - tags
+ - triggers
+ variables:
+ - $CI_COMMIT_BRANCH == "master"
+ - $CI_COMMIT_MESSAGE =~ /\[TASK\] Pre-Releasing/
+ - $CI_COMMIT_MESSAGE =~ /\[TASK\] Bump/
+
+test_py36:
+ <<: *test
+ variables:
+ TOX_ENV: "py36"
+ PYTHON_VERSION: "3.6"
+
+test_py37:
+ <<: *test
+ variables:
+ TOX_ENV: "py37"
+ PYTHON_VERSION: "3.7"
+
+test_py38:
+ <<: *test
+ variables:
+ TOX_ENV: "py38"
+ PYTHON_VERSION: "3.8"
+
+test_py39:
+ <<: *test
+ variables:
+ TOX_ENV: "py39"
+ PYTHON_VERSION: "3.9"
+
+test_py310:
+ <<: *test
+ variables:
+ TOX_ENV: "py310"
+ PYTHON_VERSION: "3.10"
 
 build_master:
  stage: build
@@ -265,18 +300,6 @@ pypi:
  - branches
  - triggers
 
-docker:
- stage: deploy
- script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
- only:
- - branches
- except:
- - [email protected]/sickrage
-
 docker_master:
  stage: deploy
  trigger:

renovate.json

 {
  "extends": [
- "config:base"
+ "config:base",
+ ":assignee(echel0n)"
  ],
- "prHourlyLimit": 2,
- "rebaseWhen": "conflicted",
  "baseBranches": [
  "develop"
  ],
+ "enabledManagers": [
+ "pip_requirements",
+ "poetry"
+ ],
  "python": {
  "commitMessageAction": "Update Python",
  "managerBranchPrefix": "py/",
- "labels": ["Update dep (Py)"]
+ "labels": [
+ "dependencies",
+ "python"
+ ]
  }
-}
+}
\ No newline at end of file

requirements-dev.txt

@@ -2,5 +2,5 @@ twine
 crowdin-cli-py
 babel
 tox
-vcrpy-unittest
 mako
+pytest
\ No newline at end of file

requirements.txt

@@ -19,7 +19,7 @@ cleverdict==1.9.2
 click==7.1.2
 cloudscraper==1.2.46
 configobj==5.0.6
-cryptography==3.2.1
+cryptography==3.3.2
 decorator==4.4.2
 deluge-client==1.9.0
 dirsync==2.2.5

setup.cfg

@@ -40,3 +40,19 @@ output_dir = sickrage/locale
 input_file = sickrage/locale/messages.pot
 ignore_obsolete = true
 previous = true
+
+[tool:pytest]
+python_files = tests/*
+filterwarnings =
+ error
+ ignore::sqlalchemy.exc.SAWarning
+ ignore::pytest.PytestUnraisableExceptionWarning
+ ignore:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning
+ ignore:There is no current event loop:DeprecationWarning
+ ignore:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning
+ ignore:ssl\.match_hostname\(\) is deprecated:DeprecationWarning
+ ignore:the load_module\(\) method is deprecated.*:DeprecationWarning
+ ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning
+ ignore:EntryPoint tuple interface is deprecated. Access members by name:DeprecationWarning
+ ignore:_SixMetaPathImporter\.exec_module\(\) not found; falling back to load_module\(\):ImportWarning
+ ignore:_SixMetaPathImporter\.find_spec\(\) not found; falling back to find_module\(\):ImportWarning

setup.py

@@ -8,13 +8,8 @@ from sickrage import __version__
 
 
 def requirements():
- with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements.txt'))) as f:
- return f.read().splitlines()
-
-
-def requirements_dev():
- with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements-dev.txt'))) as f:
- return f.read().splitlines()
+ with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'requirements.txt')) as f:
+ return f.read().splitlines(keepends=False)
 
 
 class CleanCommand(Command):
@@ -61,9 +56,6 @@ setup(
  keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'],
  packages=['sickrage'],
  install_requires=requirements(),
- extras_require={
- 'dev': requirements_dev()
- },
  include_package_data=True,
  python_requires='>=3',
  platforms='any',

sickrage/__init__.py

@@ -34,13 +34,10 @@ import gettext
 import multiprocessing
 import os
 import pathlib
-import re
 import site
-import subprocess
 import threading
 import time
 import traceback
-import pkg_resources
 
 # pywin32 for windows service
 try:
@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False):
  sys.path.extend(remainder)
 
 # set system default language
-gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
+gettext.install('messages', LOCALE_DIR, names=["ngettext"])
 
 if __install_type__ == 'windows':
  class SiCKRAGEService(win32serviceutil.ServiceFramework):
@@ -248,31 +245,6 @@ def changelog():
  return f.read()
 
 
-def check_requirements():
- if os.path.exists(REQS_FILE):
- with open(REQS_FILE) as f:
- for line in f.readlines():
- try:
- req_name, req_version = line.strip().split('==', 1)
- if 'python_version' in req_version:
- m = re.search('(\d+.\d+.\d+).*(\d+.\d+)', req_version)
- req_version = m.group(1)
- python_version = m.group(2)
- python_version_major = int(python_version.split('.')[0])
- python_version_minor = int(python_version.split('.')[1])
- if sys.version_info.major == python_version_major and sys.version_info.minor != python_version_minor:
- continue
-
- if not pkg_resources.get_distribution(req_name).version == req_version:
- print('Updating requirement {} to {}'.format(req_name, req_version))
- subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
- except pkg_resources.DistributionNotFound:
- print('Installing requirement {}'.format(line.strip()))
- subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
- except ValueError:
- continue
-
-
 def verify_checksums(remove_unverified=False):
  valid_files = []
  exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)]
@@ -484,6 +456,9 @@ def start():
  app.pid = app.daemon.pid
 
  app.start()
+
+ from tornado.ioloop import IOLoop
+ IOLoop.current().start()
  except (SystemExit, KeyboardInterrupt):
  if app:
  app.shutdown()

sickrage/core/__init__.py

@@ -256,7 +256,7 @@ class Core(object):
  self.metadata_providers = MetadataProviders()
  self.search_providers = SearchProviders()
  self.series_providers = SeriesProviders()
- self.log = Logger()
+ self.log = Logger(consoleLogging=not self.quiet, logFile=os.path.join(self.data_dir, 'logs', 'sickrage.log'))
  self.alerts = Notifications()
  self.wserver = WebServer()
  self.show_queue = ShowQueue()
@@ -347,9 +347,7 @@ class Core(object):
  # setup logger settings
  self.log.logSize = self.config.general.log_size
  self.log.logNr = self.config.general.log_nr
- self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
  self.log.debugLogging = self.debug or self.config.general.debug
- self.log.consoleLogging = not self.quiet
 
  # start logger
  self.log.start()
@@ -570,9 +568,6 @@ class Core(object):
  # perform shutdown trigger check every 5 seconds
  PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
 
- # start ioloop
- IOLoop.current().start()
-
  def init_sentry(self):
  # sentry log handler
  sentry_logging = LoggingIntegration(
@@ -732,6 +727,11 @@ class Core(object):
  # save settings
  self.config.save()
 
+ # shutdown databases
+ self.main_db.shutdown()
+ self.config.db.shutdown()
+ self.cache_db.shutdown()
+
  # shutdown logging
  if self.log:
  self.log.close()

sickrage/core/api/__init__.py

@@ -110,7 +110,7 @@ class API(object):
  def health(self):
  for i in range(3):
  try:
- health = requests.get(urljoin(self.api_base, "health"), verify=False, timeout=30).ok
+ health = requests.get(urljoin(self.api_base, "health"), verify=True, timeout=30).ok
  except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
  pass
  else:

sickrage/core/config/helpers.py

@@ -110,7 +110,7 @@ def change_gui_lang(language):
  gt.install(names=["ngettext"])
  else:
  # System default language
- gettext.install('messages', sickrage.LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
+ gettext.install('messages', sickrage.LOCALE_DIR, names=["ngettext"])
 
  sickrage.app.config.gui.gui_lang = language
 

sickrage/core/databases/__init__.py

@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs):
 
 class IntFlag(TypeDecorator):
  impl = sqlalchemy.types.Integer()
+ cache_ok = True
 
  def __init__(self, enum):
  self.enum = enum
@@ -152,20 +153,21 @@ class SRDatabase(object):
 
  @property
  def engine(self):
- if self.db_type == 'sqlite':
- return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
- elif self.db_type == 'mysql':
+ if self.db_type == 'mysql':
  mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False)
  mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}")
  return create_engine(
  'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name),
  echo=False)
+ else:
+ return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
 
  @property
  def version(self):
- context = MigrationContext.configure(self.engine)
- current_rev = context.get_current_revision()
- return current_rev
+ with self.engine.connect() as connection:
+ context = MigrationContext.configure(connection)
+ current_rev = context.get_current_revision()
+ return current_rev
 
  def setup(self):
  if inspect(self.engine).has_table('migrate_version'):
@@ -321,3 +323,6 @@ class SRDatabase(object):
  rows.append(row._asdict())
  session.bulk_insert_mappings(table, rows)
  session.commit()
+
+ def shutdown(self):
+ self.session.close()

sickrage/core/helpers/show_names.py

@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words):
  """
  if isinstance(words, str):
  words = words.split(',')
- items = [(re.compile('(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
+ items = [(re.compile(r'(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
  for regexp, word in items:
  if regexp.search(name):
  return word

sickrage/core/imdb_popular.py

@@ -95,7 +95,7 @@ class imdbPopular(object):
 
  @staticmethod
  def change_size(image_url, factor=3):
- match = re.search("^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
+ match = re.search(r"^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
 
  if match:
  matches = match.groups()

sickrage/core/logger/__init__.py

@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()):
  if self.logFile:
  # make logs folder if it doesn't exist
  if not os.path.exists(os.path.dirname(self.logFile)):
- if not make_dir(os.path.dirname(self.logFile)):
+ if not os.makedirs(os.path.dirname(self.logFile)):
  return
 
  if sickrage.app.developer:

sickrage/core/nzbSplitter.py

@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season):
 
  filename = name.replace(".nzb", "")
 
- regex = '([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
+ regex = r'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
 
  sceneNameMatch = re.search(regex, filename, re.I)
  if sceneNameMatch:
@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season):
  sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.")
  return {}, ''
 
- regex = '(' + re.escape(showName) + '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
+ regex = '(' + re.escape(showName) + r'\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
  regex = regex.replace(' ', '.')
 
  epFiles = {}
  xmlns = None
 
  for curFile in nzbElement.getchildren():
- xmlnsMatch = re.match("{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
+ xmlnsMatch = re.match(r"{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
  if not xmlnsMatch:
  continue
  else:

sickrage/core/queues/__init__.py

@@ -84,7 +84,7 @@ class Queue(object):
  ids.append(worker_id)
 
  self.auto_remove_tasks_timer = threading.Timer(10.0, self.auto_remove_tasks)
- self.auto_remove_tasks_timer.setName(self.name)
+ self.auto_remove_tasks_timer.name = self.name
  self.auto_remove_tasks_timer.start()
 
  return ids
@@ -94,12 +94,14 @@ class Queue(object):
  This function stops and kills a worker.
  If no id is provided, all workers are killed.
  :param worker_id: the identifier for the worker to kill
+ :param quiet: silence logging
  :return: None
  """
  try:
  self.lock.acquire()
  if worker_id is None:
  sickrage.app.log.info("Shutting down all {} workers".format(self.name))
+
  for worker in self.workers:
  worker.must_die = True
  else:

sickrage/core/webserver/handlers/api/__init__.py

@@ -50,11 +50,11 @@ class APIBaseHandler(RequestHandler):
 
  method_name = self.request.method.lower()
  if method_name == 'options':
- return
+ return self.finish()
 
  certs = sickrage.app.auth_server.certs()
  if not certs:
- return
+ return self.finish()
 
  auth_header = self.request.headers.get('Authorization')
 
@@ -93,7 +93,7 @@ class APIBaseHandler(RequestHandler):
  })
 
  if sickrage.app.config.user.sub_id != decoded_token.get('sub'):
- return self._unauthorized(error='user is not authorized')
+ return self.finish(self._unauthorized(error='user is not authorized'))
 
  if not sickrage.app.config.general.sso_api_key:
  sickrage.app.config.general.sso_api_key = decoded_token.get('apikey')
@@ -117,11 +117,11 @@ class APIBaseHandler(RequestHandler):
  method = self.run_async(getattr(self, method_name))
  setattr(self, method_name, method)
  except Exception:
- return self._unauthorized(error='failed to decode token')
+ self.finish(self._unauthorized(error='failed to decode token'))
  else:
- return self._unauthorized(error='invalid authorization request')
+ self.finish(self._unauthorized(error='invalid authorization request'))
  else:
- return self._unauthorized(error='authorization header missing')
+ self.finish(self._unauthorized(error='authorization header missing'))
 
  def run_async(self, method):
  @functools.wraps(method)

sickrage/core/webserver/handlers/home/__init__.py

@@ -166,7 +166,7 @@ class IsAliveHandler(BaseHandler):
  if not srcallback:
  return _("Error: Unsupported Request. Send jsonp request with 'srcallback' variable in the query string.")
 
- return "{}({})".format(srcallback, {'msg': str(sickrage.app.pid) if sickrage.app.started else 'nope'})
+ return '{}({})'.format(srcallback, {"msg": str(sickrage.app.pid) if sickrage.app.started else 'nope'})
 
 
 class TestSABnzbdHandler(BaseHandler):