Commit 3c148855 authored by echel0n's avatar echel0n
Browse files

Merge branch 'develop'

# Conflicts:
#	renovate.json
parents f145ec5e bd7d8843
......@@ -4,10 +4,6 @@
/sickrage/libs/adba/anime-list.xml
/sickrage/libs/adba/animetitles.xml
# SR Tox Files #
######################
/.tox/
# SR GitLab Files #
######################
/.gitlab/
......@@ -20,9 +16,9 @@
# SR User Related #
######################
*.db*
*.ini
*.torrent
*.magnet
config.ini
swagger.json
privatekey.pem
autoProcessTV.cfg
......@@ -33,11 +29,9 @@ autoProcessTV.cfg
# SR Test Related #
######################
/tests/database/
/tests/providers/
/tests/show name/
/tests/show name final/
/tests/sr_update_*/
/tests/data/
/.tox/
report.xml
# Compiled Source #
######################
......
......@@ -2,6 +2,7 @@ stages:
# - review_webpack
# - review_docker
# - review_deploy
- test
- build
- sentry
- deploy
......@@ -99,27 +100,61 @@ stages:
# only:
# - [email protected]/sickrage
build:
stage: build
retry: 2
.test_template: &test
stage: test
retry: 1
image:
name: nikolaik/python-nodejs:python3.9-nodejs10-alpine
name: python:$PYTHON_VERSION
variables:
NODE_ENV: "development"
CARGO_HOME: "$CI_PROJECT_DIR/cargo"
ASYNC_TEST_TIMEOUT: 60
script:
- export PATH="$CARGO_HOME/bin:$PATH"
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache
- pip install -U pip
- pip install -r requirements-dev.txt
- yarn run build
- python checksum-generator.py
only:
- branches
- pip install tox
- tox -e $TOX_ENV
artifacts:
when: always
reports:
junit: report.xml
paths:
- report.xml
expire_in: 1 week
except:
- [email protected]/sickrage
refs:
- tags
- triggers
variables:
- $CI_COMMIT_BRANCH == "master"
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Pre-Releasing/
- $CI_COMMIT_MESSAGE =~ /\[TASK\] Bump/
test_py36:
<<: *test
variables:
TOX_ENV: "py36"
PYTHON_VERSION: "3.6"
test_py37:
<<: *test
variables:
TOX_ENV: "py37"
PYTHON_VERSION: "3.7"
test_py38:
<<: *test
variables:
TOX_ENV: "py38"
PYTHON_VERSION: "3.8"
test_py39:
<<: *test
variables:
TOX_ENV: "py39"
PYTHON_VERSION: "3.9"
test_py310:
<<: *test
variables:
TOX_ENV: "py310"
PYTHON_VERSION: "3.10"
build_master:
stage: build
......@@ -265,18 +300,6 @@ pypi:
- branches
- triggers
docker:
stage: deploy
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
only:
- branches
except:
- [email protected]/sickrage
docker_master:
stage: deploy
trigger:
......
{
"extends": [
"config:base"
"config:base",
":assignee(echel0n)"
],
"prHourlyLimit": 2,
"rebaseWhen": "conflicted",
"baseBranches": [
"develop"
],
"enabledManagers": [
"pip_requirements",
"poetry"
],
"python": {
"commitMessageAction": "Update Python",
"managerBranchPrefix": "py/",
"labels": ["Update dep (Py)"]
"labels": [
"dependencies",
"python"
]
}
}
}
\ No newline at end of file
......@@ -2,5 +2,5 @@ twine
crowdin-cli-py
babel
tox
vcrpy-unittest
mako
pytest
\ No newline at end of file
......@@ -19,7 +19,7 @@ cleverdict==1.9.2
click==7.1.2
cloudscraper==1.2.46
configobj==5.0.6
cryptography==3.2.1
cryptography==3.3.2
decorator==4.4.2
deluge-client==1.9.0
dirsync==2.2.5
......
......@@ -40,3 +40,19 @@ output_dir = sickrage/locale
input_file = sickrage/locale/messages.pot
ignore_obsolete = true
previous = true
[tool:pytest]
python_files = tests/*
filterwarnings =
error
ignore::sqlalchemy.exc.SAWarning
ignore::pytest.PytestUnraisableExceptionWarning
ignore:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning
ignore:There is no current event loop:DeprecationWarning
ignore:ssl\.PROTOCOL_TLS is deprecated:DeprecationWarning
ignore:ssl\.match_hostname\(\) is deprecated:DeprecationWarning
ignore:the load_module\(\) method is deprecated.*:DeprecationWarning
ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning
ignore:EntryPoint tuple interface is deprecated. Access members by name:DeprecationWarning
ignore:_SixMetaPathImporter\.exec_module\(\) not found; falling back to load_module\(\):ImportWarning
ignore:_SixMetaPathImporter\.find_spec\(\) not found; falling back to find_module\(\):ImportWarning
......@@ -8,13 +8,8 @@ from sickrage import __version__
def requirements():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements.txt'))) as f:
return f.read().splitlines()
def requirements_dev():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'requirements-dev.txt'))) as f:
return f.read().splitlines()
with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'requirements.txt')) as f:
return f.read().splitlines(keepends=False)
class CleanCommand(Command):
......@@ -61,9 +56,6 @@ setup(
keywords=['sickrage', 'sickragetv', 'tv', 'torrent', 'nzb', 'video', 'echel0n'],
packages=['sickrage'],
install_requires=requirements(),
extras_require={
'dev': requirements_dev()
},
include_package_data=True,
python_requires='>=3',
platforms='any',
......
......@@ -34,13 +34,10 @@ import gettext
import multiprocessing
import os
import pathlib
import re
import site
import subprocess
import threading
import time
import traceback
import pkg_resources
# pywin32 for windows service
try:
......@@ -76,7 +73,7 @@ if not (LIBS_DIR in sys.path) and not getattr(sys, 'frozen', False):
sys.path.extend(remainder)
# set system default language
gettext.install('messages', LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
gettext.install('messages', LOCALE_DIR, names=["ngettext"])
if __install_type__ == 'windows':
class SiCKRAGEService(win32serviceutil.ServiceFramework):
......@@ -248,31 +245,6 @@ def changelog():
return f.read()
def check_requirements():
if os.path.exists(REQS_FILE):
with open(REQS_FILE) as f:
for line in f.readlines():
try:
req_name, req_version = line.strip().split('==', 1)
if 'python_version' in req_version:
m = re.search('(\d+.\d+.\d+).*(\d+.\d+)', req_version)
req_version = m.group(1)
python_version = m.group(2)
python_version_major = int(python_version.split('.')[0])
python_version_minor = int(python_version.split('.')[1])
if sys.version_info.major == python_version_major and sys.version_info.minor != python_version_minor:
continue
if not pkg_resources.get_distribution(req_name).version == req_version:
print('Updating requirement {} to {}'.format(req_name, req_version))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except pkg_resources.DistributionNotFound:
print('Installing requirement {}'.format(line.strip()))
subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", line.strip()])
except ValueError:
continue
def verify_checksums(remove_unverified=False):
valid_files = []
exempt_files = [pathlib.Path(__file__), pathlib.Path(CHECKSUM_FILE), pathlib.Path(AUTO_PROCESS_TV_CFG_FILE)]
......@@ -484,6 +456,9 @@ def start():
app.pid = app.daemon.pid
app.start()
from tornado.ioloop import IOLoop
IOLoop.current().start()
except (SystemExit, KeyboardInterrupt):
if app:
app.shutdown()
......
......@@ -256,7 +256,7 @@ class Core(object):
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.series_providers = SeriesProviders()
self.log = Logger()
self.log = Logger(consoleLogging=not self.quiet, logFile=os.path.join(self.data_dir, 'logs', 'sickrage.log'))
self.alerts = Notifications()
self.wserver = WebServer()
self.show_queue = ShowQueue()
......@@ -347,9 +347,7 @@ class Core(object):
# setup logger settings
self.log.logSize = self.config.general.log_size
self.log.logNr = self.config.general.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.debug or self.config.general.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
......@@ -570,9 +568,6 @@ class Core(object):
# perform shutdown trigger check every 5 seconds
PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
# start ioloop
IOLoop.current().start()
def init_sentry(self):
# sentry log handler
sentry_logging = LoggingIntegration(
......@@ -732,6 +727,11 @@ class Core(object):
# save settings
self.config.save()
# shutdown databases
self.main_db.shutdown()
self.config.db.shutdown()
self.cache_db.shutdown()
# shutdown logging
if self.log:
self.log.close()
......
......@@ -110,7 +110,7 @@ class API(object):
def health(self):
for i in range(3):
try:
health = requests.get(urljoin(self.api_base, "health"), verify=False, timeout=30).ok
health = requests.get(urljoin(self.api_base, "health"), verify=True, timeout=30).ok
except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
pass
else:
......
......@@ -110,7 +110,7 @@ def change_gui_lang(language):
gt.install(names=["ngettext"])
else:
# System default language
gettext.install('messages', sickrage.LOCALE_DIR, codeset='UTF-8', names=["ngettext"])
gettext.install('messages', sickrage.LOCALE_DIR, names=["ngettext"])
sickrage.app.config.gui.gui_lang = language
......
......@@ -66,6 +66,7 @@ def instant_defaults_listener(target, args, kwargs):
class IntFlag(TypeDecorator):
impl = sqlalchemy.types.Integer()
cache_ok = True
def __init__(self, enum):
self.enum = enum
......@@ -152,20 +153,21 @@ class SRDatabase(object):
@property
def engine(self):
if self.db_type == 'sqlite':
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
elif self.db_type == 'mysql':
if self.db_type == 'mysql':
mysql_engine = create_engine('mysql+pymysql://{}:{}@{}:{}/'.format(self.db_username, self.db_password, self.db_host, self.db_port), echo=False)
mysql_engine.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_prefix}_{self.name}")
return create_engine(
'mysql+pymysql://{}:{}@{}:{}/{}_{}'.format(self.db_username, self.db_password, self.db_host, self.db_port, self.db_prefix, self.name),
echo=False)
else:
return create_engine('sqlite:///{}'.format(self.db_path), echo=False, connect_args={'check_same_thread': False, 'timeout': 30})
@property
def version(self):
context = MigrationContext.configure(self.engine)
current_rev = context.get_current_revision()
return current_rev
with self.engine.connect() as connection:
context = MigrationContext.configure(connection)
current_rev = context.get_current_revision()
return current_rev
def setup(self):
if inspect(self.engine).has_table('migrate_version'):
......@@ -321,3 +323,6 @@ class SRDatabase(object):
rows.append(row._asdict())
session.bulk_insert_mappings(table, rows)
session.commit()
def shutdown(self):
self.session.close()
......@@ -60,7 +60,7 @@ def contains_at_least_one_word(name, words):
"""
if isinstance(words, str):
words = words.split(',')
items = [(re.compile('(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
items = [(re.compile(r'(^|[\W_])%s($|[\W_])' % re.escape(word.strip()), re.I), word.strip()) for word in words]
for regexp, word in items:
if regexp.search(name):
return word
......
......@@ -95,7 +95,7 @@ class imdbPopular(object):
@staticmethod
def change_size(image_url, factor=3):
match = re.search("^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
match = re.search(r"^(.*)V1_(.{2})(.*?)_(.{2})(.*?),(.*?),(.*?),(.\d?)_(.*?)_.jpg$", image_url)
if match:
matches = match.groups()
......
......@@ -157,7 +157,7 @@ class Logger(logging.getLoggerClass()):
if self.logFile:
# make logs folder if it doesn't exist
if not os.path.exists(os.path.dirname(self.logFile)):
if not make_dir(os.path.dirname(self.logFile)):
if not os.makedirs(os.path.dirname(self.logFile)):
return
if sickrage.app.developer:
......
......@@ -45,7 +45,7 @@ def getSeasonNZBs(name, urlData, season):
filename = name.replace(".nzb", "")
regex = '([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
regex = r'([\w\._\ ]+)[\. ]S%02d[\. ]([\w\._\-\ ]+)[\- ]([\w_\-\ ]+?)' % season
sceneNameMatch = re.search(regex, filename, re.I)
if sceneNameMatch:
......@@ -54,14 +54,14 @@ def getSeasonNZBs(name, urlData, season):
sickrage.app.log.error("Unable to parse " + name + " into a scene name. If it's a valid, log a bug.")
return {}, ''
regex = '(' + re.escape(showName) + '\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
regex = '(' + re.escape(showName) + r'\.S%02d(?:[E0-9]+)\.[\w\._]+\-\w+' % season + ')'
regex = regex.replace(' ', '.')
epFiles = {}
xmlns = None
for curFile in nzbElement.getchildren():
xmlnsMatch = re.match("{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
xmlnsMatch = re.match(r"{(http://[A-Za-z0-9_./]+/nzb)\}file", curFile.tag)
if not xmlnsMatch:
continue
else:
......
......@@ -84,7 +84,7 @@ class Queue(object):
ids.append(worker_id)
self.auto_remove_tasks_timer = threading.Timer(10.0, self.auto_remove_tasks)
self.auto_remove_tasks_timer.setName(self.name)
self.auto_remove_tasks_timer.name = self.name
self.auto_remove_tasks_timer.start()
return ids
......@@ -94,12 +94,14 @@ class Queue(object):
This function stops and kills a worker.
If no id is provided, all workers are killed.
:param worker_id: the identifier for the worker to kill
:param quiet: silence logging
:return: None
"""
try:
self.lock.acquire()
if worker_id is None:
sickrage.app.log.info("Shutting down all {} workers".format(self.name))
for worker in self.workers:
worker.must_die = True
else:
......
......@@ -50,11 +50,11 @@ class APIBaseHandler(RequestHandler):
method_name = self.request.method.lower()
if method_name == 'options':
return
return self.finish()
certs = sickrage.app.auth_server.certs()
if not certs:
return
return self.finish()
auth_header = self.request.headers.get('Authorization')
......@@ -93,7 +93,7 @@ class APIBaseHandler(RequestHandler):
})
if sickrage.app.config.user.sub_id != decoded_token.get('sub'):
return self._unauthorized(error='user is not authorized')
return self.finish(self._unauthorized(error='user is not authorized'))
if not sickrage.app.config.general.sso_api_key:
sickrage.app.config.general.sso_api_key = decoded_token.get('apikey')
......@@ -117,11 +117,11 @@ class APIBaseHandler(RequestHandler):
method = self.run_async(getattr(self, method_name))
setattr(self, method_name, method)
except Exception:
return self._unauthorized(error='failed to decode token')
self.finish(self._unauthorized(error='failed to decode token'))
else:
return self._unauthorized(error='invalid authorization request')
self.finish(self._unauthorized(error='invalid authorization request'))
else:
return self._unauthorized(error='authorization header missing')
self.finish(self._unauthorized(error='authorization header missing'))
def run_async(self, method):
@functools.wraps(method)
......
......@@ -166,7 +166,7 @@ class IsAliveHandler(BaseHandler):
if not srcallback:
return _("Error: Unsupported Request. Send jsonp request with 'srcallback' variable in the query string.")
return "{}({})".format(srcallback, {'msg': str(sickrage.app.pid) if sickrage.app.started else 'nope'})
return '{}({})'.format(srcallback, {"msg": str(sickrage.app.pid) if sickrage.app.started else 'nope'})
class TestSABnzbdHandler(BaseHandler):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment