Commit b5a95fd6 authored by echel0n's avatar echel0n
Browse files

Refactored external API to v6

Added schedule for updating web server ssl certificates once a day
Added JWT token auth to web socket server
Added SiCKRAGE AMQP messaging
Refactored version updates to be handled by AMQP
Refactored network timezone updates to be handled by AMQP
Refactored search provider url updates to be handled by AMQP
Refactored how application version is stored and accessed
Refactored app update system
Added automated gitlab releasing to CI script
Refactored torrent trackers to come from SR API
parent 8a3b26b4
stages:
# - review_webpack
# - review_docker
# - review_deploy
# - review_webpack
# - review_docker
# - review_deploy
- release_build
- release_sentry
- release_deploy
- release_publish
#review:webpack:
# stage: review_webpack
......@@ -98,6 +99,28 @@ stages:
# only:
# - [email protected]/sickrage
release_build:
stage: release_build
retry: 2
image:
name: nikolaik/python-nodejs:python3.9-nodejs10-alpine
variables:
NODE_ENV: "development"
CARGO_HOME: "$CI_PROJECT_DIR/cargo"
script:
- export PATH="$CARGO_HOME/bin:$PATH"
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- export ENABLE_SENTRY_RELEASE="false" && yarn install --pure-lockfile --cache-folder .yarn-cache
- pip install -U pip
- pip install -r requirements-dev.txt
- yarn run build
- python checksum-generator.py
only:
- branches
except:
- [email protected]/sickrage
release_build_master:
stage: release_build
image:
......@@ -107,15 +130,15 @@ release_build_master:
CARGO_HOME: "$CI_PROJECT_DIR/cargo"
script:
- export PATH="$CARGO_HOME/bin:$PATH"
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl
- apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl unzip
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- git config --global user.email $(git --no-pager show -s --format='%ae' HEAD)
- git config --global user.name $(git --no-pager show -s --format='%an' HEAD)
- pip install -U pip
- pip install bumpversion
- pip install -r requirements-dev.txt
- RELEASE_VERSION=$(bumpversion --dry-run --list release | awk -F= '/new_version=/ { print $2 }')
- bumpversion --allow-dirty release package.json sickrage/version.txt
- bumpversion --allow-dirty release package.json sickrage/version.txt sickrage/__init__.py
- RELEASE_VERSION=${grep __version__ sickrage/__init__.py | grep -o '[0-9][^"]*'}
- npx auto-changelog -v $RELEASE_VERSION --hide-credit --package --commit-limit false --ignore-commit-pattern \[TASK\].*
- yarn install --pure-lockfile --cache-folder .yarn-cache
- yarn run build
......@@ -131,10 +154,11 @@ release_build_master:
- git push https://$GIT_ACCESS_USER:[email protected]$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:master --follow-tags
- git checkout develop
- git merge --ff-only release/$RELEASE_VERSION
- bumpversion --allow-dirty patch package.json sickrage/version.txt
- bumpversion --allow-dirty patch package.json sickrage/version.txt sickrage/__init__.py
- RELEASE_VERSION=${grep __version__ sickrage/__init__.py | grep -o '[0-9][^"]*'}
- python checksum-generator.py
- git add --all
- git commit -m "[TASK] Bump develop branch to v$(cat sickrage/version.txt)"
- git commit -m "[TASK] Bump develop branch to v$RELEASE_VERSION"
- git push https://$GIT_ACCESS_USER:[email protected]$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:develop --follow-tags
when: manual
only:
......@@ -162,8 +186,9 @@ release_build_develop:
- pip install -U pip
- pip install bumpversion
- pip install -r requirements-dev.txt
- bumpversion --allow-dirty dev package.json sickrage/version.txt
- npx auto-changelog -v $(cat sickrage/version.txt) --hide-credit --unreleased --package --commit-limit false --ignore-commit-pattern \[TASK\].*
- bumpversion --allow-dirty dev package.json sickrage/version.txt sickrage/__init__.py
- RELEASE_VERSION=${grep __version__ sickrage/__init__.py | grep -o '[0-9][^"]*'}
- npx auto-changelog -v $RELEASE_VERSION --hide-credit --unreleased --package --commit-limit false --ignore-commit-pattern \[TASK\].*
- yarn run build
- python checksum-generator.py
# - python setup.py extract_messages
......@@ -173,8 +198,8 @@ release_build_develop:
- git config --global user.email $(git --no-pager show -s --format='%ae' HEAD)
- git config --global user.name $(git --no-pager show -s --format='%an' HEAD)
- git add --all
- git commit -m "[TASK] Pre-Releasing v$(cat sickrage/version.txt)"
- git tag -a $(cat sickrage/version.txt) -m "Pre-release v$(cat sickrage/version.txt)"
- git commit -m "[TASK] Pre-Releasing v$RELEASE_VERSION"
- git tag -a $RELEASE_VERSION -m "Pre-release v$RELEASE_VERSION"
- git push https://$GIT_ACCESS_USER:[email protected]$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:$CI_COMMIT_REF_NAME --follow-tags
only:
- [email protected]/sickrage
......@@ -191,16 +216,17 @@ release_sentry_master:
retry: 2
image:
name: getsentry/sentry-cli
entrypoint: [""]
entrypoint: [ "" ]
script:
- export SENTRY_URL=$SENTRY_URL
- export SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
- export SENTRY_ORG=$SENTRY_ORG
- export SENTRY_PROJECT=$SENTRY_PROJECT
- sentry-cli releases new --project $SENTRY_PROJECT $(cat sickrage/version.txt)
- sentry-cli releases set-commits --auto $(cat sickrage/version.txt)
- sentry-cli releases finalize $(cat sickrage/version.txt)
- sentry-cli releases deploys $(cat sickrage/version.txt) new -e master
- RELEASE_VERSION=${grep __version__ sickrage/__init__.py | grep -o '[0-9][^"]*'}
- sentry-cli releases new --project $SENTRY_PROJECT $RELEASE_VERSION
- sentry-cli releases set-commits --auto $RELEASE_VERSION
- sentry-cli releases finalize $RELEASE_VERSION
- sentry-cli releases deploys $RELEASE_VERSION new -e master
only:
- /^[0-9.]+$/@SiCKRAGE/sickrage
except:
......@@ -212,22 +238,33 @@ release_sentry_develop:
retry: 2
image:
name: getsentry/sentry-cli
entrypoint: [""]
entrypoint: [ "" ]
script:
- export SENTRY_URL=$SENTRY_URL
- export SENTRY_AUTH_TOKEN=$SENTRY_AUTH_TOKEN
- export SENTRY_ORG=$SENTRY_ORG
- export SENTRY_PROJECT=$SENTRY_PROJECT
- sentry-cli releases new --project $SENTRY_PROJECT $(cat sickrage/version.txt)
- sentry-cli releases set-commits --auto $(cat sickrage/version.txt)
- sentry-cli releases finalize $(cat sickrage/version.txt)
- sentry-cli releases deploys $(cat sickrage/version.txt) new -e develop
- RELEASE_VERSION=${grep __version__ sickrage/__init__.py | grep -o '[0-9][^"]*'}
- sentry-cli releases new --project $SENTRY_PROJECT $RELEASE_VERSION
- sentry-cli releases set-commits --auto $RELEASE_VERSION
- sentry-cli releases finalize $RELEASE_VERSION
- sentry-cli releases deploys $RELEASE_VERSION new -e develop
only:
- /^[0-9.]+dev[0-9]+$/@SiCKRAGE/sickrage
except:
- branches
- triggers
release_publish:
stage: release_publish
image: registry.gitlab.com/gitlab-org/release-cli:latest
script:
- release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG
only:
- tags
rules:
- if: $CI_COMMIT_TAG
deploy_pypi:
stage: release_deploy
retry: 2
......@@ -240,6 +277,7 @@ deploy_pypi:
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- pip install -U pip
- pip install -U twine
- sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"pip\"/" sickrage/__init__.py
- python setup.py sdist bdist_wheel
- twine upload dist/*
only:
......@@ -249,6 +287,18 @@ deploy_pypi:
- branches
- triggers
deploy_docker:
stage: release_deploy
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- sed -i "s/^__install_type__ = [\"']\(.*\)[\"']/__install_type__ = \"docker\"/" sickrage/__init__.py
- docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
only:
- branches
except:
- [email protected]/sickrage
deploy_docker_master:
stage: release_deploy
trigger:
......
syntax = "proto3";
package app.protobufs.v1;
message CreatedAnnouncementResponse {
string ahash = 1;
string title = 2;
string description = 3;
string image = 4;
string date = 5;
}
message DeletedAnnouncementResponse {
string ahash = 1;
}
syntax = "proto3";
package app.protobufs.v1;
message SavedNetworkTimezoneResponse {
string network = 1;
string timezone = 2;
}
message DeletedNetworkTimezoneResponse {
string network = 1;
}
syntax = "proto3";
package app.protobufs.v1;
message SavedSearchProviderUrlResponse {
string provider_id = 1;
string provider_urls = 2;
}
syntax = "proto3";
package app.protobufs.v1;
message SavedServerCertificateResponse {
string certificate = 1;
string private_key = 2;
}
syntax = "proto3";
package app.protobufs.v1;
message UpdatedAppResponse {
bool force = 1;
}
......@@ -53,8 +53,10 @@ oauth2==1.9.0.post1
oauthlib==3.1.0
packaging==20.4
pbr==5.4.5
pika==1.2.0
Pint==0.14
profilehooks==1.11.2
protobuf==3.17.3
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.20
......
import glob
import os
import shutil
import glob
from setuptools import setup, Command
def version():
with open(os.path.abspath(os.path.join(os.path.dirname(__file__), 'sickrage', 'version.txt'))) as f:
return f.read()
from sickrage import __version__
def requirements():
......@@ -55,7 +52,7 @@ except ImportError:
setup(
name='sickrage',
version=version(),
version=__version__,
description='Automatic Video Library Manager for TV Shows',
author='echel0n',
author_email='[email protected]',
......
......@@ -19,6 +19,8 @@
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
__version__ = "10.0.12.dev4"
__install_type__ = ""
import argparse
import atexit
......@@ -41,7 +43,6 @@ MAIN_DIR = os.path.abspath(os.path.realpath(os.path.expanduser(os.path.dirname(o
PROG_DIR = os.path.abspath(os.path.realpath(os.path.expanduser(os.path.dirname(__file__))))
LOCALE_DIR = os.path.join(PROG_DIR, 'locale')
LIBS_DIR = os.path.join(PROG_DIR, 'libs')
VERSION_FILE = os.path.join(PROG_DIR, 'version.txt')
CHANGELOG_FILE = os.path.join(MAIN_DIR, 'CHANGELOG.md')
REQS_FILE = os.path.join(MAIN_DIR, 'requirements.txt')
CHECKSUM_FILE = os.path.join(PROG_DIR, 'checksums.md5')
......@@ -227,8 +228,7 @@ def file_cleanup(remove=False):
def version():
# Get the version number
with open(VERSION_FILE) as f:
return f.read()
return __version__
def changelog():
......@@ -259,7 +259,7 @@ def main():
parser = argparse.ArgumentParser(prog='sickrage')
parser.add_argument('-v', '--version',
action='version',
version='%(prog)s {}'.format(version()))
version=version())
parser.add_argument('-d', '--daemon',
action='store_true',
help='Run as a daemon (*NIX ONLY)')
......@@ -399,8 +399,6 @@ def main():
# start app
app.start()
while app.started:
time.sleep(0.1)
except (SystemExit, KeyboardInterrupt):
if app:
app.shutdown()
......
......@@ -38,14 +38,16 @@ from urllib.request import FancyURLopener
import rarfile
import sentry_sdk
from apscheduler.schedulers import SchedulerNotRunningError
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.schedulers.tornado import TornadoScheduler
from apscheduler.triggers.interval import IntervalTrigger
from dateutil import tz
from fake_useragent import UserAgent
from sentry_sdk.integrations.logging import LoggingIntegration
from tornado.ioloop import IOLoop
from tornado.platform.asyncio import AnyThreadEventLoopPolicy
import sickrage
from sickrage.core.amqp import AMQPClient
from sickrage.core.announcements import Announcements
from sickrage.core.api import API
from sickrage.core.auth import AuthServer
......@@ -57,7 +59,7 @@ from sickrage.core.databases.config import ConfigDB, CustomStringEncryptedType
from sickrage.core.databases.main import MainDB
from sickrage.core.enums import MultiEpNaming, DefaultHomePage, NzbMethod, TorrentMethod, CheckPropersInterval
from sickrage.core.helpers import generate_secret, make_dir, restore_app_data, get_disk_space_usage, get_free_space, launch_browser, torrent_webui_url, \
encryption, md5_file_hash, flatten
encryption, md5_file_hash, flatten, get_internal_ip
from sickrage.core.logger import Logger
from sickrage.core.nameparser.validator import check_force_season_folders
from sickrage.core.processors import auto_postprocessor
......@@ -145,7 +147,7 @@ class Core(object):
self.db_username = None
self.db_password = None
self.debug = None
self.newest_version_string = None
self.latest_version_string = None
self.naming_ep_type = (
"%(seasonnumber)dx%(episodenumber)02d",
......@@ -227,6 +229,7 @@ class Core(object):
self.auth_server = None
self.announcements = None
self.api = None
self.amqp_client = None
def start(self):
self.started = True
......@@ -241,7 +244,7 @@ class Core(object):
self.init_sentry()
# scheduler
self.scheduler = BackgroundScheduler({'apscheduler.timezone': 'UTC'})
self.scheduler = TornadoScheduler({'apscheduler.timezone': 'UTC'})
# init core classes
self.api = API()
......@@ -271,6 +274,7 @@ class Core(object):
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.announcements = Announcements()
self.amqp_client = AMQPClient()
# authorization sso client
self.auth_server = AuthServer()
......@@ -318,13 +322,13 @@ class Core(object):
self.config.migrate_config_file(self.config_file)
# add server id tag to sentry
sentry_sdk.set_tag('server_id', sickrage.app.config.general.server_id)
sentry_sdk.set_tag('server_id', self.config.general.server_id)
# add user to sentry
sentry_sdk.set_user({
'id': sickrage.app.config.user.sub_id,
'username': sickrage.app.config.user.username,
'email': sickrage.app.config.user.email
'id': self.config.user.sub_id,
'username': self.config.user.username,
'email': self.config.user.email
})
# config overrides
......@@ -401,28 +405,6 @@ class Core(object):
if self.config.general.show_update_hour < 0 or self.config.general.show_update_hour > 23:
self.config.general.show_update_hour = 0
# add version checker job
self.scheduler.add_job(
self.version_updater.task,
IntervalTrigger(
hours=self.config.general.version_updater_freq,
timezone='utc'
),
name=self.version_updater.name,
id=self.version_updater.name
)
# add network timezones updater job
self.scheduler.add_job(
self.tz_updater.task,
IntervalTrigger(
days=1,
timezone='utc'
),
name=self.tz_updater.name,
id=self.tz_updater.name
)
# add show updater job
self.scheduler.add_job(
self.show_updater.task,
......@@ -534,26 +516,15 @@ class Core(object):
id=self.upnp_client.name
)
# add announcements job
self.scheduler.add_job(
self.announcements.task,
IntervalTrigger(
minutes=15,
timezone='utc'
),
name=self.announcements.name,
id=self.announcements.name
)
# add provider URL update job
# add server connections update job
self.scheduler.add_job(
self.search_providers.task,
self.api.update_server_connections,
IntervalTrigger(
hours=1,
days=1,
timezone='utc'
),
name=self.search_providers.name,
id=self.search_providers.name
name=self.api.name,
id=self.api.name
)
# start queues
......@@ -564,17 +535,32 @@ class Core(object):
# start web server
self.wserver.start()
# fire off jobs now
self.scheduler.get_job(self.version_updater.name).modify(next_run_time=datetime.datetime.utcnow())
self.scheduler.get_job(self.tz_updater.name).modify(next_run_time=datetime.datetime.utcnow())
self.scheduler.get_job(self.announcements.name).modify(next_run_time=datetime.datetime.utcnow())
self.scheduler.get_job(self.search_providers.name).modify(next_run_time=datetime.datetime.utcnow())
# update server connections
self.scheduler.get_job(self.api.name).modify(next_run_time=datetime.datetime.utcnow())
# start scheduler service
self.scheduler.start()
# load shows
self.scheduler.add_job(self.load_shows)
IOLoop.current().add_callback(self.load_shows)
# load network timezones
IOLoop.current().spawn_callback(self.tz_updater.update_network_timezones)
# load search provider urls
IOLoop.current().spawn_callback(self.search_providers.update_urls)
# startup message
IOLoop.current().add_callback(self.startup_message)
# launch browser
IOLoop.current().add_callback(self.launch_browser)
# shutdown trigger
IOLoop.current().add_callback(self.shutdown_trigger)
# start ioloop
IOLoop.current().start()
def init_sentry(self):
# sentry log handler
......@@ -633,6 +619,22 @@ class Core(object):
self.log.info('Loading initial shows list finished')
def startup_message(self):
self.log.info("SiCKRAGE :: STARTED")
self.log.info(f"SiCKRAGE :: APP VERSION:[{sickrage.version()}]")
self.log.info(f"SiCKRAGE :: CONFIG VERSION:[v{self.config.db.version}]")
self.log.info(f"SiCKRAGE :: DATABASE VERSION:[v{self.main_db.version}]")
self.log.info(f"SiCKRAGE :: DATABASE TYPE:[{self.db_type}]")
self.log.info(f"SiCKRAGE :: INSTALL TYPE:[{self.version_updater.updater.type}]")
self.log.info(
f"SiCKRAGE :: URL:[{('http', 'https')[self.config.general.enable_https]}://{(get_internal_ip(), self.web_host)[self.web_host != '']}:{self.config.general.web_port}/{self.config.general.web_root}]")
def launch_browser(self):
if not self.no_launch and self.config.general.launch_browser:
launch_browser(protocol=('http', 'https')[self.config.general.enable_https],
host=(get_internal_ip(), self.web_host)[self.web_host != ''],
startport=self.config.general.web_port)
def shutdown(self, restart=False):
if self.started:
self.log.info('SiCKRAGE IS {}!!!'.format(('SHUTTING DOWN', 'RESTARTING')[restart]))
......@@ -653,6 +655,9 @@ class Core(object):
self.show_queue.shutdown()
self.postprocessor_queue.shutdown()
# stop amqp consumer
self.amqp_client.stop()
# log out of ADBA
if self.adba_connection:
self.log.debug("Shutting down ANIDB connection")
......@@ -673,7 +678,13 @@ class Core(object):
if restart:
os.execl(sys.executable, sys.executable, *sys.argv)
if sickrage.app.daemon:
sickrage.app.daemon.stop()
if self.daemon:
self.daemon.stop()
self.started = False
def shutdown_trigger(self):
if self.started:
IOLoop.current().add_timeout(datetime.timedelta(seconds=5), self.shutdown_trigger)
else:
IOLoop.current().stop()
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
import urllib.parse
import pika
from google.protobuf.json_format import MessageToDict
from pika.adapters.tornado_connection import TornadoConnection
from tornado.ioloop import IOLoop
import sickrage
from sickrage.protos.announcement_v1_pb2 import CreatedAnnouncementResponse, DeletedAnnouncementResponse
from sickrage.protos.network_timezone_v1_pb2 import SavedNetworkTimezoneResponse, DeletedNetworkTimezoneResponse
from sickrage.protos.search_provider_url_v1_pb2 import SavedSearchProviderUrlResponse
from sickrage.protos.server_certificate_v1_pb2 import SavedServerCertificateResponse
from sickrage.protos.updates_v1_pb2 import UpdatedAppResponse
class AMQPClient(object):
def __init__(self):
self._amqp_host = 'rmq.sickrage.ca'
self._amqp_port = 5671
self._amqp_vhost = 'sickrage-app'
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
IOLoop.current().spawn_callback(self.connect)
@property
def events(self):
return {
'server_ssl_certificate.saved': {
'event_type': SavedServerCertificateResponse(),
'event_cmd': sickrage.app.wserver.load_ssl_certificate,
},
'network_timezone.saved': {
'event_type': SavedNetworkTimezoneResponse(),
'event_cmd': sickrage.app.tz_updater.update_network_timezone,
},
'network_timezone.deleted': {
'event_type': DeletedNetworkTimezoneResponse(),
'event_cmd': sickrage.app.tz_updater.delete_network_timezone,
},
'search_provider_url.saved': {
'event_type': SavedSearchProviderUrlResponse(),
'event_cmd': sickrage.app.search_providers.update_url,
},
'app.updated': {
'event_type': UpdatedAppResponse(),
'event_cmd': sickrage.app.version_updater.task,
},
'announcement.created': {
'event_type': CreatedAnnouncementResponse(),
'event_cmd': sickrage.app.announcements.add,
},
'announcement.deleted': {
'event_type': DeletedAnnouncementResponse(),
'event_cmd': sickrage.app.announcements.clear,
},
}