Commit bd7d8843 authored by echel0n's avatar echel0n
Browse files

bumped cryptography dependency

modified renovate.json
made sqlite the default database type
refactored unit tests
added params to silence logging during shutdown
added test stage to CI using pytest and tox
updated .gitignore
fixed issues with quiet console logging
removed all messages.json files
parent 1522609d
...@@ -4,10 +4,6 @@ ...@@ -4,10 +4,6 @@
/sickrage/libs/adba/anime-list.xml /sickrage/libs/adba/anime-list.xml
/sickrage/libs/adba/animetitles.xml /sickrage/libs/adba/animetitles.xml
# SR Tox Files #
######################
/.tox/
# SR GitLab Files # # SR GitLab Files #
###################### ######################
/.gitlab/ /.gitlab/
...@@ -20,9 +16,9 @@ ...@@ -20,9 +16,9 @@
# SR User Related # # SR User Related #
###################### ######################
*.db* *.db*
*.ini
*.torrent *.torrent
*.magnet *.magnet
config.ini
swagger.json swagger.json
privatekey.pem privatekey.pem
autoProcessTV.cfg autoProcessTV.cfg
...@@ -33,11 +29,9 @@ autoProcessTV.cfg ...@@ -33,11 +29,9 @@ autoProcessTV.cfg
# SR Test Related # # SR Test Related #
###################### ######################
/tests/database/ /tests/data/
/tests/providers/ /.tox/
/tests/show name/ report.xml
/tests/show name final/
/tests/sr_update_*/
# Compiled Source # # Compiled Source #
###################### ######################
......
...@@ -2,6 +2,7 @@ stages: ...@@ -2,6 +2,7 @@ stages:
# - review_webpack # - review_webpack
# - review_docker # - review_docker
# - review_deploy # - review_deploy
- test
- build - build
- sentry - sentry
- deploy - deploy
...@@ -99,27 +100,61 @@ stages: ...@@ -99,27 +100,61 @@ stages:
# only: # only:
# - [email protected]/sickrage # - [email protected]/sickrage
build: .test_template: &test
stage: build stage: test
retry: 2 retry: 1
image: image:
name: nikolaik/python-nodejs:python3.9-nodejs10-alpine name: python:$PYTHON_VERSION
variables: variables:
NODE_ENV: "development" ASYNC_TEST_TIMEOUT: 60
CARGO_HOME: "$CI_PROJECT_DIR/cargo"
script: script:
- export PATH="$CARGO_HOME/bin:$PATH" - pip install tox
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip - tox -e $TOX_ENV
- curl https://sh.rustup.rs -sSf | sh -s -- -y artifacts:
- export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache when: always
- pip install -U pip reports:
- pip install -r requirements-dev.txt junit: report.xml
- yarn run build paths:
- python checksum-generator.py - report.xml
only: expire_in: 1 week
- branches
except: except:
- [email protected]/sickrage refs:
- tags
- triggers
variables:
- $CI_COMMIT_BRANCH == "master"
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Pre-Releasing/
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Bump/
test_py36:
<<: *test
variables:
TOX_ENV: "py36"
PYTHON_VERSION: "3.6"
test_py37:
<<: *test
variables:
TOX_ENV: "py37"
PYTHON_VERSION: "3.7"
test_py38:
<<: *test
variables:
TOX_ENV: "py38"
PYTHON_VERSION: "3.8"
test_py39:
<<: *test
variables:
TOX_ENV: "py39"
PYTHON_VERSION: "3.9"
test_py310:
<<: *test
variables:
TOX_ENV: "py310"
PYTHON_VERSION: "3.10"
build_master: build_master:
stage: build stage: build
...@@ -265,18 +300,6 @@ pypi: ...@@ -265,18 +300,6 @@ pypi:
- branches - branches
- triggers - triggers
docker:
stage: deploy
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
only:
- branches
except:
- [email protected]/sickrage
docker_master: docker_master:
stage: deploy stage: deploy
trigger: trigger:
......
{ {
"extends": [ "extends": [
"config:base" "config:base",
":assignee(echel0n)"
], ],
"prHourlyLimit": 2,
"rebaseWhen": "conflicted",
"baseBranches": [ "baseBranches": [
"develop" "develop"
], ],
"enabledManagers": [
"pip_requirements",
"poetry"
],
"python": { "python": {
"commitMessageAction": "Update Python", "commitMessageAction": "Update Python",
"managerBranchPrefix": "py/", "managerBranchPrefix": "py/",
"labels": ["Update dep (Py)"] "labels": [
"dependencies",
"python"
]
} }
} }
\ No newline at end of file
...@@ -2,5 +2,5 @@ twine ...@@ -2,5 +2,5 @@ twine
crowdin-cli-py crowdin-cli-py
babel babel
tox tox
vcrpy-unittest
mako mako
pytest
\ No newline at end of file
...@@ -19,7 +19,7 @@ cleverdict==1.9.2 ...@@ -19,7 +19,7 @@ cleverdict==1.9.2
click==7.1.2 click==7.1.2
cloudscraper==1.2.46 cloudscraper==1.2.46
configobj==5.0.6 configobj==5.0.6
cryptography==3.2.1 cryptography==3.3.2
decorator==4.4.2 decorator==4.4.2
deluge-client==1.9.0 deluge-client==1.9.0
dirsync==2.2.5 dirsync==2.2.5
......
...@@ -40,3 +40,19 @@ output_dir = sickrage/locale ...@@ -40,3 +40,19 @@ output_dir = sickrage/locale
input_file = sickrage/locale/messages.pot input_file = sickrage/locale/messages.pot
ignore_obsolete = true ignore_obsolete = true
previous = true previous = true
[tool:pytest]
python_files = tests/*
filterwarnings =
error
ignore::sqlalchemy.exc.SAWarning
ignore::pytest.PytestUnraisableExceptionWarning
ignore:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning
ignore:There is no current event loop:DeprecationWarning
ignore:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning
ignore:ssl\.match_hostname\(\) is deprecated:DeprecationWarning
ignore:the load_module\(\) method is deprecated.*:DeprecationWarning
ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning
ignore:EntryPoint tuple interface is deprecated. Access members by name:DeprecationWarning
ignore:_SixMetaPathImporter\.exec_module\(\) not found; falling back to load_module\(\):ImportWarning
ignore:_SixMetaPathImporter\.find_spec\(\) not found; falling back to find_module\(\):ImportWarning
...@@ -8,13 +8,8 @@ from sickrage import __version__ ...@@ -8,13 +8,8 @@ from sickrage import __version__
def requirements(): def requirements():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements.txt'))) as f: with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'requirements.txt')) as f:
return f.read().splitlines() return f.read().splitlines(keepends=False)
def requirements_dev():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements-dev.txt'))) as f:
return f.read().splitlines()
class CleanCommand(Command): class CleanCommand(Command):
...@@ -61,9 +56,6 @@ setup( ...@@ -61,9 +56,6 @@ setup(
keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'], keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'],
packages=['sickrage'], packages=['sickrage'],
install_requires=requirements(), install_requires=requirements(),
extras_require={
'dev': requirements_dev()
},
include_package_data=True, include_package_data=True,
python_requires='>=3', python_requires='>=3',
platforms='any', platforms='any',
......
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
...@@ -34,13 +34,10 @@ import gettext ...@@ -34,13 +34,10 @@ import gettext
import multiprocessing import multiprocessing
import os import os
import pathlib import pathlib
import re
import site import site
import subprocess
import threading import threading
import time import time
import traceback import traceback
import pkg_resources
# pywin32 for windows service # pywin32 for windows service
try: try:
...@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False): ...@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False):
sys.path.extend(remainder) sys.path.extend(remainder)
# set system default language # set system default language
gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"]) gettext.install('messages', LOCALE_DIR, names=["ngettext"])
if __install_type__ == 'windows': if __install_type__ == 'windows':
class SiCKRAGEService(win32serviceutil.ServiceFramework): class SiCKRAGEService(win32serviceutil.ServiceFramework):
...@@ -248,31 +245,6 @@ def changelog(): ...@@ -248,31 +245,6 @@ def changelog():
return f.read() return f.read()
def check_requirements():
if os.path.exists(REQS_FILE):
with open(REQS_FILE) as f:
for line in f.readlines():
try:
req_name, req_version = line.strip().split('==', 1)
if 'python_version' in req_version:
m = re.search('(\d+.\d+.\d+).*(\d+.\d+)', req_version)
req_version = m.group(1)
python_version = m.group(2)
python_version_major = int(python_version.split('.')[0])
python_version_minor = int(python_version.split('.')[1])
if sys.version_info.major == python_version_major and sys.version_info.minor != python_version_minor:
continue
if not pkg_resources.get_distribution(req_name).version == req_version:
print('Updating requirement {} to {}'.format(req_name, req_version))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except pkg_resources.DistributionNotFound:
print('Installing requirement {}'.format(line.strip()))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except ValueError:
continue
def verify_checksums(remove_unverified=False): def verify_checksums(remove_unverified=False):
valid_files = [] valid_files = []
exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)] exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)]
...@@ -484,6 +456,9 @@ def start(): ...@@ -484,6 +456,9 @@ def start():
app.pid = app.daemon.pid app.pid = app.daemon.pid
app.start() app.start()
from tornado.ioloop import IOLoop
IOLoop.current().start()
except (SystemExit, KeyboardInterrupt): except (SystemExit, KeyboardInterrupt):
if app: if app:
app.shutdown() app.shutdown()
......
...@@ -256,7 +256,7 @@ class Core(object): ...@@ -256,7 +256,7 @@ class Core(object):
self.metadata_providers = MetadataProviders() self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders() self.search_providers = SearchProviders()
self.series_providers = SeriesProviders() self.series_providers = SeriesProviders()
self.log = Logger() self.log = Logger(consoleLogging=not self.quiet, logFile=os.path.join(self.data_dir, 'logs', 'sickrage.log'))
self.alerts = Notifications() self.alerts = Notifications()
self.wserver = WebServer() self.wserver = WebServer()
self.show_queue = ShowQueue() self.show_queue = ShowQueue()
...@@ -347,9 +347,7 @@ class Core(object): ...@@ -347,9 +347,7 @@ class Core(object):
# setup logger settings # setup logger settings
self.log.logSize = self.config.general.log_size self.log.logSize = self.config.general.log_size
self.log.logNr = self.config.general.log_nr self.log.logNr = self.config.general.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.debug or self.config.general.debug self.log.debugLogging = self.debug or self.config.general.debug
self.log.consoleLogging = not self.quiet
# start logger # start logger
self.log.start() self.log.start()
...@@ -570,9 +568,6 @@ class Core(object): ...@@ -570,9 +568,6 @@ class Core(object):
# perform shutdown trigger check every 5 seconds # perform shutdown trigger check every 5 seconds
PeriodicCallback(self.shutdown_trigger, 5 * 1000).start() PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
# start ioloop
IOLoop.current().start()
def init_sentry(self): def init_sentry(self):
# sentry log handler # sentry log handler
sentry_logging = LoggingIntegration( sentry_logging = LoggingIntegration(
...@@ -732,6 +727,11 @@ class Core(object): ...@@ -732,6 +727,11 @@ class Core(object):
# save settings # save settings
self.config.save() self.config.save()
# shutdown databases
self.main_db.shutdown()
self.config.db.shutdown()
self.cache_db.shutdown()
# shutdown logging # shutdown logging
if self.log: if self.log:
self.log.close() self.log.close()
......
...@@ -110,7 +110,7 @@ class API(object): ...@@ -110,7 +110,7 @@ class API(object):
def health(self): def health(self):
for i in range(3): for i in range(3):
try: try:
health = requests.get(urljoin(self.api_base, "health"), verify=False, timeout=30).ok health = requests.get(urljoin(self.api_base, "health"), verify=True, timeout=30).ok
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
pass pass
else: else:
......
...@@ -110,7 +110,7 @@ def change_gui_lang(language): ...@@ -110,7 +110,7 @@ def change_gui_lang(language):
gt.install(names=["ngettext"]) gt.install(names=["ngettext"])
else: else:
# System default language # System default language
gettext.install('messages', sickrage.LOCALE_DIR, codeset='UTF-8', names=["ngettext"]) gettext.install('messages', sickrage.LOCALE_DIR, names=["ngettext"])
sickrage.app.config.gui.gui_lang = language sickrage.app.config.gui.gui_lang = language
......
...@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs): ...@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs):
class IntFlag(TypeDecorator): class IntFlag(TypeDecorator):
impl = sqlalchemy.types.Integer() impl = sqlalchemy.types.Integer()
cache_ok = True
def __init__(self, enum): def __init__(self, enum):
self.enum = enum self.enum = enum
...@@ -152,20 +153,21 @@ class SRDatabase(object): ...@@ -152,20 +153,21 @@ class SRDatabase(object):
@property @property
def engine(self): def engine(self):
if self.db_type == 'sqlite': if self.db_type == 'mysql':
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
elif self.db_type == 'mysql':
mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False) mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False)
mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}") mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}")
return create_engine( return create_engine(
'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name), 'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name),
echo=False) echo=False)
else:
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
@property @property
def version(self): def version(self):
context = MigrationContext.configure(self.engine) with self.engine.connect() as connection:
current_rev = context.get_current_revision() context = MigrationContext.configure(connection)
return current_rev current_rev = context.get_current_revision()
return current_rev
def setup(self): def setup(self):
if inspect(self.engine).has_table('migrate_version'): if inspect(self.engine).has_table('migrate_version'):
...@@ -321,3 +323,6 @@ class SRDatabase(object): ...@@ -321,3 +323,6 @@ class SRDatabase(object):
rows.append(row._asdict()) rows.append(row._asdict())
session.bulk_insert_mappings(table, rows) session.bulk_insert_mappings(table, rows)
session.commit() session.commit()
def shutdown(self):
self.session.close()
...@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words): ...@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words):
""" """
if isinstance(words, str): if isinstance(words, str):
words = words.split(',') words = words.split(',')
items = [(re.compile('(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words] items = [(re.compile(r'(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
for regexp, word in items: for regexp, word in items:
if regexp.search(name): if regexp.search(name):
return word return word
......
...@@ -95,7 +95,7 @@ class imdbPopular(object): ...@@ -95,7 +95,7 @@ class imdbPopular(object):
@staticmethod @staticmethod
def change_size(image_url, factor=3): def change_size(image_url, factor=3):
match = re.search("^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url) match = re.search(r"^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
if match: if match:
matches = match.groups() matches = match.groups()
......
...@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()): ...@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()):
if self.logFile: if self.logFile:
# make logs folder if it doesn't exist # make logs folder if it doesn't exist
if not os.path.exists(os.path.dirname(self.logFile)): if not os.path.exists(os.path.dirname(self.logFile)):
if not make_dir(os.path.dirname(self.logFile)): if not os.makedirs(os.path.dirname(self.logFile)):
return return
if sickrage.app.developer: if sickrage.app.developer:
......
...@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season): ...@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season):
filename = name.replace(".nzb", "") filename = name.replace(".nzb", "")
regex = '([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season regex = r'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
sceneNameMatch = re.search(regex, filename, re.I) sceneNameMatch = re.search(regex, filename, re.I)
if sceneNameMatch: if sceneNameMatch:
...@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season): ...@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season):
sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.") sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.")
return {}, '' return {}, ''
regex = '(' + re.escape(showName) + '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')' regex = '(' + re.escape(showName) + r'\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
regex = regex.replace(' ', '.') regex = regex.replace(' ', '.')
epFiles = {} epFiles = {}
xmlns = None xmlns = None
for curFile in nzbElement.getchildren(): for curFile in nzbElement.getchildren():
xmlnsMatch = re.match("{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag) xmlnsMatch = re.match(r"{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
if not xmlnsMatch: if not xmlnsMatch:
continue continue
else: else:
......
...@@ -84,7 +84,7 @@ class Queue(object): ...@@ -84,7 +84,7 @@ class Queue(object):
ids.append(worker_id) ids.append(worker_id)