Commit bd7d8843 authored by echel0n's avatar echel0n
Browse files

bumped cryptography dependency

modified renovate.json
made sqlite the default database type
refactored unit tests
added params to silence logging during shutdown
added test stage to CI using pytest and tox
updated .gitignore
fixed issues with quiet console logging
removed all messages.json files
parent 1522609d
......@@ -4,10 +4,6 @@
/sickrage/libs/adba/anime-list.xml
/sickrage/libs/adba/animetitles.xml
# SR Tox Files #
######################
/.tox/
# SR GitLab Files #
######################
/.gitlab/
......@@ -20,9 +16,9 @@
# SR User Related #
######################
*.db*
*.ini
*.torrent
*.magnet
config.ini
swagger.json
privatekey.pem
autoProcessTV.cfg
......@@ -33,11 +29,9 @@ autoProcessTV.cfg
# SR Test Related #
######################
/tests/database/
/tests/providers/
/tests/show name/
/tests/show name final/
/tests/sr_update_*/
/tests/data/
/.tox/
report.xml
# Compiled Source #
######################
......
......@@ -2,6 +2,7 @@ stages:
# - review_webpack
# - review_docker
# - review_deploy
- test
- build
- sentry
- deploy
......@@ -99,27 +100,61 @@ stages:
# only:
# - [email protected]/sickrage
build:
stage: build
retry: 2
.test_template: &test
stage: test
retry: 1
image:
name: nikolaik/python-nodejs:python3.9-nodejs10-alpine
name: python:$PYTHON_VERSION
variables:
NODE_ENV: "development"
CARGO_HOME: "$CI_PROJECT_DIR/cargo"
ASYNC_TEST_TIMEOUT: 60
script:
- export PATH="$CARGO_HOME/bin:$PATH"
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache
- pip install -U pip
- pip install -r requirements-dev.txt
- yarn run build
- python checksum-generator.py
only:
- branches
- pip install tox
- tox -e $TOX_ENV
artifacts:
when: always
reports:
junit: report.xml
paths:
- report.xml
expire_in: 1 week
except:
- [email protected]/sickrage
refs:
- tags
- triggers
variables:
- $CI_COMMIT_BRANCH == "master"
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Pre-Releasing/
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Bump/
test_py36:
<<: *test
variables:
TOX_ENV: "py36"
PYTHON_VERSION: "3.6"
test_py37:
<<: *test
variables:
TOX_ENV: "py37"
PYTHON_VERSION: "3.7"
test_py38:
<<: *test
variables:
TOX_ENV: "py38"
PYTHON_VERSION: "3.8"
test_py39:
<<: *test
variables:
TOX_ENV: "py39"
PYTHON_VERSION: "3.9"
test_py310:
<<: *test
variables:
TOX_ENV: "py310"
PYTHON_VERSION: "3.10"
build_master:
stage: build
......@@ -265,18 +300,6 @@ pypi:
- branches
- triggers
docker:
stage: deploy
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
only:
- branches
except:
- [email protected]/sickrage
docker_master:
stage: deploy
trigger:
......
{
"extends": [
"config:base"
"config:base",
":assignee(echel0n)"
],
"prHourlyLimit": 2,
"rebaseWhen": "conflicted",
"baseBranches": [
"develop"
],
"enabledManagers": [
"pip_requirements",
"poetry"
],
"python": {
"commitMessageAction": "Update Python",
"managerBranchPrefix": "py/",
"labels": ["Update dep (Py)"]
"labels": [
"dependencies",
"python"
]
}
}
}
\ No newline at end of file
......@@ -2,5 +2,5 @@ twine
crowdin-cli-py
babel
tox
vcrpy-unittest
mako
pytest
\ No newline at end of file
......@@ -19,7 +19,7 @@ cleverdict==1.9.2
click==7.1.2
cloudscraper==1.2.46
configobj==5.0.6
cryptography==3.2.1
cryptography==3.3.2
decorator==4.4.2
deluge-client==1.9.0
dirsync==2.2.5
......
......@@ -40,3 +40,19 @@ output_dir = sickrage/locale
input_file = sickrage/locale/messages.pot
ignore_obsolete = true
previous = true
[tool:pytest]
python_files = tests/*
filterwarnings =
error
ignore::sqlalchemy.exc.SAWarning
ignore::pytest.PytestUnraisableExceptionWarning
ignore:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning
ignore:There is no current event loop:DeprecationWarning
ignore:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning
ignore:ssl\.match_hostname\(\) is deprecated:DeprecationWarning
ignore:the load_module\(\) method is deprecated.*:DeprecationWarning
ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning
ignore:EntryPoint tuple interface is deprecated. Access members by name:DeprecationWarning
ignore:_SixMetaPathImporter\.exec_module\(\) not found; falling back to load_module\(\):ImportWarning
ignore:_SixMetaPathImporter\.find_spec\(\) not found; falling back to find_module\(\):ImportWarning
......@@ -8,13 +8,8 @@ from sickrage import __version__
def requirements():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements.txt'))) as f:
return f.read().splitlines()
def requirements_dev():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements-dev.txt'))) as f:
return f.read().splitlines()
with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'requirements.txt')) as f:
return f.read().splitlines(keepends=False)
class CleanCommand(Command):
......@@ -61,9 +56,6 @@ setup(
keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'],
packages=['sickrage'],
install_requires=requirements(),
extras_require={
'dev': requirements_dev()
},
include_package_data=True,
python_requires='>=3',
platforms='any',
......
......@@ -34,13 +34,10 @@ import gettext
import multiprocessing
import os
import pathlib
import re
import site
import subprocess
import threading
import time
import traceback
import pkg_resources
# pywin32 for windows service
try:
......@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False):
sys.path.extend(remainder)
# set system default language
gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
gettext.install('messages', LOCALE_DIR, names=["ngettext"])
if __install_type__ == 'windows':
class SiCKRAGEService(win32serviceutil.ServiceFramework):
......@@ -248,31 +245,6 @@ def changelog():
return f.read()
def check_requirements():
if os.path.exists(REQS_FILE):
with open(REQS_FILE) as f:
for line in f.readlines():
try:
req_name, req_version = line.strip().split('==', 1)
if 'python_version' in req_version:
m = re.search('(\d+.\d+.\d+).*(\d+.\d+)', req_version)
req_version = m.group(1)
python_version = m.group(2)
python_version_major = int(python_version.split('.')[0])
python_version_minor = int(python_version.split('.')[1])
if sys.version_info.major == python_version_major and sys.version_info.minor != python_version_minor:
continue
if not pkg_resources.get_distribution(req_name).version == req_version:
print('Updating requirement {} to {}'.format(req_name, req_version))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except pkg_resources.DistributionNotFound:
print('Installing requirement {}'.format(line.strip()))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except ValueError:
continue
def verify_checksums(remove_unverified=False):
valid_files = []
exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)]
......@@ -484,6 +456,9 @@ def start():
app.pid = app.daemon.pid
app.start()
from tornado.ioloop import IOLoop
IOLoop.current().start()
except (SystemExit, KeyboardInterrupt):
if app:
app.shutdown()
......
......@@ -256,7 +256,7 @@ class Core(object):
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.series_providers = SeriesProviders()
self.log = Logger()
self.log = Logger(consoleLogging=not self.quiet, logFile=os.path.join(self.data_dir, 'logs', 'sickrage.log'))
self.alerts = Notifications()
self.wserver = WebServer()
self.show_queue = ShowQueue()
......@@ -347,9 +347,7 @@ class Core(object):
# setup logger settings
self.log.logSize = self.config.general.log_size
self.log.logNr = self.config.general.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.debug or self.config.general.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
......@@ -570,9 +568,6 @@ class Core(object):
# perform shutdown trigger check every 5 seconds
PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
# start ioloop
IOLoop.current().start()
def init_sentry(self):
# sentry log handler
sentry_logging = LoggingIntegration(
......@@ -732,6 +727,11 @@ class Core(object):
# save settings
self.config.save()
# shutdown databases
self.main_db.shutdown()
self.config.db.shutdown()
self.cache_db.shutdown()
# shutdown logging
if self.log:
self.log.close()
......
......@@ -110,7 +110,7 @@ class API(object):
def health(self):
for i in range(3):
try:
health = requests.get(urljoin(self.api_base, "health"), verify=False, timeout=30).ok
health = requests.get(urljoin(self.api_base, "health"), verify=True, timeout=30).ok
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
pass
else:
......
......@@ -110,7 +110,7 @@ def change_gui_lang(language):
gt.install(names=["ngettext"])
else:
# System default language
gettext.install('messages', sickrage.LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
gettext.install('messages', sickrage.LOCALE_DIR, names=["ngettext"])
sickrage.app.config.gui.gui_lang = language
......
......@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs):
class IntFlag(TypeDecorator):
impl = sqlalchemy.types.Integer()
cache_ok = True
def __init__(self, enum):
self.enum = enum
......@@ -152,20 +153,21 @@ class SRDatabase(object):
@property
def engine(self):
if self.db_type == 'sqlite':
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
elif self.db_type == 'mysql':
if self.db_type == 'mysql':
mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False)
mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}")
return create_engine(
'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name),
echo=False)
else:
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
@property
def version(self):
context = MigrationContext.configure(self.engine)
current_rev = context.get_current_revision()
return current_rev
with self.engine.connect() as connection:
context = MigrationContext.configure(connection)
current_rev = context.get_current_revision()
return current_rev
def setup(self):
if inspect(self.engine).has_table('migrate_version'):
......@@ -321,3 +323,6 @@ class SRDatabase(object):
rows.append(row._asdict())
session.bulk_insert_mappings(table, rows)
session.commit()
def shutdown(self):
self.session.close()
......@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words):
"""
if isinstance(words, str):
words = words.split(',')
items = [(re.compile('(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
items = [(re.compile(r'(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
for regexp, word in items:
if regexp.search(name):
return word
......
......@@ -95,7 +95,7 @@ class imdbPopular(object):
@staticmethod
def change_size(image_url, factor=3):
match = re.search("^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
match = re.search(r"^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
if match:
matches = match.groups()
......
......@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()):
if self.logFile:
# make logs folder if it doesn't exist
if not os.path.exists(os.path.dirname(self.logFile)):
if not make_dir(os.path.dirname(self.logFile)):
if not os.makedirs(os.path.dirname(self.logFile)):
return
if sickrage.app.developer:
......
......@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season):
filename = name.replace(".nzb", "")
regex = '([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
regex = r'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
sceneNameMatch = re.search(regex, filename, re.I)
if sceneNameMatch:
......@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season):
sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.")
return {}, ''
regex = '(' + re.escape(showName) + '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
regex = '(' + re.escape(showName) + r'\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
regex = regex.replace(' ', '.')
epFiles = {}
xmlns = None
for curFile in nzbElement.getchildren():
xmlnsMatch = re.match("{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
xmlnsMatch = re.match(r"{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
if not xmlnsMatch:
continue
else:
......
......@@ -84,7 +84,7 @@ class Queue(object):
ids.append(worker_id)
self.auto_remove_tasks_timer = threading.Timer(10.0, self.auto_remove_tasks)
self.auto_remove_tasks_timer.setName(self.name)
self.auto_remove_tasks_timer.name = self.name
self.auto_remove_tasks_timer.start()
return ids
......@@ -94,12 +94,14 @@ class Queue(object):
This function stops and kills a worker.
If no id is provided, all workers are killed.
:param worker_id: the identifier for the worker to kill
:param quiet: silence logging
:return: None
"""
try:
self.lock.acquire()
if worker_id is None:
sickrage.app.log.info("Shutting down all {} workers".format(self.name))
for worker in self.workers:
worker.must_die = True
else:
......
......@@ -50,11 +50,11 @@ class APIBaseHandler(RequestHandler):
method_name = self.request.method.lower()
if method_name == 'options':
return
return self.finish()
certs = sickrage.app.auth_server.certs()
if not certs:
return
return self.finish()
auth_header = self.request.headers.get('Authorization')
......@@ -93,7 +93,7 @@ class APIBaseHandler(RequestHandler):
})
if sickrage.app.config.user.sub_id != decoded_token.get('sub'):
return self._unauthorized(error='user is not authorized')
return self.finish(self._unauthorized(error='user is not authorized'))
if not sickrage.app.config.general.sso_api_key:
sickrage.app.config.general.sso_api_key = decoded_token.get('apikey')
......@@ -117,11 +117,11 @@ class APIBaseHandler(RequestHandler):
method = self.run_async(getattr(self, method_name))
setattr(self, method_name, method)
except Exception:
return self._unauthorized(error='failed to decode token')
self.finish(self._unauthorized(error='failed to decode token'))
else:
return self._unauthorized(error='invalid authorization request')
self.finish(self._unauthorized(error='invalid authorization request'))
else:
return self._unauthorized(error='authorization header missing')
self.finish(self._unauthorized(error='authorization header missing'))
def run_async(self, method):
@functools.wraps(method)
......
......@@ -166,7 +166,7 @@ class IsAliveHandler(BaseHandler):
if not srcallback:
return _("Error: Unsupported Request. Send jsonp request with 'srcallback' variable in the query string.")
return "{}({})".format(srcallback, {'msg': str(sickrage.app.pid) if sickrage.app.started else 'nope'})
return '{}({})'.format(srcallback, {"msg": str(sickrage.app.pid) if sickrage.app.started else 'nope'})
class TestSABnzbdHandler(BaseHandler):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment