Commit 973d96d8 authored by echel0n's avatar echel0n

Added dirsync to requirements.txt

Refactored source update's to sync and purge files that no longer exist
Added checksum generator and validator to help maintain file integrity of project
Fixed #SICKRAGE-APP-5Q6 - AttributeError in sickrage.core.version_updater in update method, 'NoneType' object has no attribute 'content'
Added fix for dupe scene absolute numbering
Fixed #SICKRAGE-APP-5R0 - AttributeError in sickrage.providers.torrent.extratorrent
parent 06591728
......@@ -105,7 +105,7 @@ services:
release:build:master:
stage: release_build
image:
name: nikolaik/python-nodejs:python3.7-nodejs10-alpine
name: nikolaik/python-nodejs:python3.8-nodejs10-alpine
variables:
NODE_ENV: "development"
script:
......@@ -118,6 +118,7 @@ release:build:master:
- bumpversion --allow-dirty release package.json sickrage/version.txt
- git checkout -b release-$(cat sickrage/version.txt)
- yarn run build
- python checksum-generator.py
- git add --all
- git commit -m "[TASK] Releasing v$(cat sickrage/version.txt)"
- git fetch . release-$(cat sickrage/version.txt):master
......@@ -126,6 +127,7 @@ release:build:master:
- git push https://$GIT_ACCESS_USER:[email protected]$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:master --follow-tags
- git checkout develop
- bumpversion --allow-dirty patch package.json sickrage/version.txt
- python checksum-generator.py
- git add --all
- git commit -m "[TASK] Bump develop branch to v$(cat sickrage/version.txt)"
- git push https://$GIT_ACCESS_USER:[email protected]$CI_SERVER_HOST/$CI_PROJECT_PATH.git HEAD:develop --follow-tags
......@@ -142,7 +144,7 @@ release:build:master:
release:build:develop:
stage: release_build
image:
name: nikolaik/python-nodejs:python3.7-nodejs10-alpine
name: nikolaik/python-nodejs:python3.8-nodejs10-alpine
variables:
NODE_ENV: "development"
script:
......@@ -153,6 +155,7 @@ release:build:develop:
- npx git-changelog -t $(git describe --abbrev=0)
- bumpversion --allow-dirty dev package.json sickrage/version.txt
- yarn run build
- python checksum-generator.py
# - python setup.py extract_messages
# - crowdin-cli-py upload sources
# - crowdin-cli-py download
......@@ -215,7 +218,7 @@ release:sentry:develop:
deploy:pypi:
stage: release_deploy
image: python:3.7-alpine3.9
image: python:3.8-alpine3.9
script:
- apk add --no-cache py-pip gcc libffi-dev python3-dev musl-dev openssl-dev
- pip install -U twine
......
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
import hashlib
import os
from pathlib import Path
main_dir = Path(__file__).parent
prog_dir = main_dir.joinpath('sickrage')
checksum_file = prog_dir.joinpath('checksums.md5')
def md5(filename):
with open(filename, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()
with open(checksum_file, "wb") as fp:
for root, dirs, files in os.walk(prog_dir):
for file in files:
full_filename = Path(str(root).replace(str(prog_dir), 'sickrage')).joinpath(file)
if full_filename != checksum_file:
fp.write('{} = {}\n'.format(full_filename, md5(full_filename)).encode())
print('Finished generating {}'.format(checksum_file))
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
import hashlib
import os
from pathlib import Path
main_dir = Path(__file__).parent
prog_dir = main_dir.joinpath('sickrage')
checksum_file = prog_dir.joinpath('checksums.md5')
def md5(filename):
with open(filename, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()
with open(checksum_file, "rb") as fp:
failed = False
for line in fp.readlines():
file, checksum = line.decode().strip().split(' = ')
full_filename = main_dir.joinpath(file)
if full_filename != checksum_file:
if not os.path.exists(full_filename) or md5(full_filename) != checksum:
print('SiCKRAGE file {} integrity check failed'.format(full_filename))
failed = True
if not failed:
print('SiCKRAGE file integrity check passed')
......@@ -45,6 +45,7 @@ sqlalchemy==1.3.18
sqlalchemy-migrate==0.13.0
mutagen==1.44.0
deluge-client==1.9.0
dirsync==2.2.4
PyMySQL
certifi
pyasn1
......
This diff is collapsed.
......@@ -49,7 +49,7 @@ from sickrage.core.config import Config
from sickrage.core.databases.cache import CacheDB
from sickrage.core.databases.main import MainDB
from sickrage.core.helpers import generate_secret, make_dir, get_lan_ip, restore_app_data, get_disk_space_usage, get_free_space, launch_browser, \
torrent_webui_url, encryption
torrent_webui_url, encryption, md5_file_hash
from sickrage.core.logger import Logger
from sickrage.core.nameparser.validator import check_force_season_folders
from sickrage.core.processors import auto_postprocessor
......@@ -70,7 +70,7 @@ from sickrage.core.updaters.rsscache_updater import RSSCacheUpdater
from sickrage.core.updaters.show_updater import ShowUpdater
from sickrage.core.updaters.tz_updater import TimeZoneUpdater
from sickrage.core.upnp import UPNPClient
from sickrage.core.version_updater import VersionUpdater
from sickrage.core.version_updater import VersionUpdater, SourceUpdateManager
from sickrage.core.webserver import WebServer
from sickrage.metadata import MetadataProviders
from sickrage.notifiers import NotifierProviders
......@@ -522,7 +522,7 @@ class Core(object):
# launch browser window
if all([not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.scheduler.add_job(launch_browser, args=[('http', 'https')[sickrage.app.config.enable_https],
sickrage.app.config.web_host, sickrage.app.config.web_port])
sickrage.app.config.web_host, sickrage.app.config.web_port])
self.log.info("SiCKRAGE :: STARTED")
self.log.info("SiCKRAGE :: APP VERSION:[{}]".format(sickrage.version()))
......@@ -530,9 +530,9 @@ class Core(object):
self.log.info("SiCKRAGE :: DATABASE VERSION:[v{}]".format(self.main_db.version))
self.log.info("SiCKRAGE :: DATABASE TYPE:[{}]".format(self.db_type))
self.log.info("SiCKRAGE :: URL:[{}://{}:{}/{}]".format(('http', 'https')[self.config.enable_https],
(self.config.web_host, get_lan_ip())[self.config.web_host == '0.0.0.0'],
self.config.web_port,
self.config.web_root))
(self.config.web_host, get_lan_ip())[self.config.web_host == '0.0.0.0'],
self.config.web_port,
self.config.web_root))
def load_shows(self):
threading.currentThread().setName('CORE')
......
......@@ -135,6 +135,34 @@ class MainDB(SRDatabase):
result.scene_episode = -1
session.commit()
def fix_duplicate_episode_scene_absolute_numbering():
session = self.session()
duplicates = session.query(
self.TVEpisode.showid,
self.TVEpisode.scene_absolute_number,
func.count(self.TVEpisode.showid).label('count')
).group_by(
self.TVEpisode.showid,
self.TVEpisode.scene_absolute_number
).filter(
self.TVEpisode.scene_absolute_number != -1
).having(literal_column('count') > 1)
for cur_duplicate in duplicates:
sickrage.app.log.debug("Duplicate episode scene absolute numbering detected! "
"showid: {dupe_id} "
"scene absolute number: {dupe_scene_absolute_number} "
"count: {dupe_count}".format(dupe_id=cur_duplicate.showid,
dupe_scene_absolute_number=cur_duplicate.scene_absolute_number,
dupe_count=cur_duplicate.count))
for result in session.query(self.TVEpisode).filter_by(showid=cur_duplicate.showid,
scene_absolute_number=cur_duplicate.scene_absolute_number).\
limit(cur_duplicate.count - 1):
result.scene_absolute_number = -1
session.commit()
def remove_invalid_episodes():
session = self.session()
......@@ -189,6 +217,7 @@ class MainDB(SRDatabase):
remove_invalid_episodes()
fix_invalid_scene_numbering()
fix_duplicate_episode_scene_numbering()
fix_duplicate_episode_scene_absolute_numbering()
fix_tvshow_table_columns()
class TVShow(MainDBBase):
......
......@@ -1744,10 +1744,11 @@ def strip_accents(name):
def md5_file_hash(filename):
hasher = hashlib.md5()
with open(filename, 'rb') as fd:
hasher.update(fd.read())
return hasher.hexdigest()
with open(filename, "rb") as f:
file_hash = hashlib.md5()
while chunk := f.read(8192):
file_hash.update(chunk)
return file_hash.hexdigest()
def get_extension(filename):
......
......@@ -30,6 +30,8 @@ import tempfile
import threading
from time import sleep
import dirsync as dirsync
import sickrage
from sickrage.core.helpers import backup_app_data
from sickrage.core.websession import WebSession
......@@ -626,7 +628,12 @@ class SourceUpdateManager(UpdateManager):
with tempfile.TemporaryFile() as update_tarfile:
sickrage.app.log.info("Downloading update from " + repr(tar_download_url))
update_tarfile.write(WebSession().get(tar_download_url).content)
resp = WebSession().get(tar_download_url)
if not resp or not resp.content:
sickrage.app.log.warning('Failed to download SiCKRAGE update')
return False
update_tarfile.write(resp.content)
update_tarfile.seek(0)
with tempfile.TemporaryDirectory(prefix='sr_update_', dir=sickrage.app.data_dir) as unpack_dir:
......@@ -639,29 +646,13 @@ class SourceUpdateManager(UpdateManager):
sickrage.app.log.warning("Invalid update data, update failed: not a gzip file")
return False
# find update dir name
update_dir_contents = [x for x in os.listdir(unpack_dir) if os.path.isdir(os.path.join(unpack_dir, x))]
if len(update_dir_contents) != 1:
sickrage.app.log.warning("Invalid update data, update failed: " + str(update_dir_contents))
if len(os.listdir(unpack_dir)) != 1:
sickrage.app.log.warning("Invalid update data, update failed")
return False
# walk temp folder and move files to main folder
content_dir = os.path.join(unpack_dir, update_dir_contents[0])
sickrage.app.log.info("Moving files from " + content_dir + " to " + sickrage.MAIN_DIR)
for dirname, __, filenames in os.walk(content_dir):
dirname = dirname[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, dirname, curfile)
new_path = os.path.join(sickrage.MAIN_DIR, dirname, curfile)
if os.path.isfile(new_path) and os.path.exists(new_path):
os.remove(new_path)
try:
shutil.move(old_path, new_path)
except IOError:
os.makedirs(os.path.dirname(new_path))
shutil.move(old_path, new_path)
update_dir = os.path.join(*[unpack_dir, os.listdir(unpack_dir)[0], 'sickrage'])
sickrage.app.log.info("Sync folder {} to {}".format(update_dir, sickrage.PROG_DIR))
dirsync.sync(update_dir, sickrage.PROG_DIR, 'sync', purge=True)
except Exception as e:
sickrage.app.log.error("Error while trying to update: {}".format(e))
return False
......
......@@ -53,7 +53,7 @@ class BaseHandler(RequestHandler, ABC):
request_info = ''.join(["<strong>%s</strong>: %s<br>" % (k, self.request.__dict__[k]) for k in self.request.__dict__.keys()])
error = exc_info[1]
sickrage.app.log.error(error)
sickrage.app.log.debug(error)
self.set_header('Content-Type', 'text/html')
return self.write("""<html>
......
......@@ -85,9 +85,12 @@ class ExtraTorrentProvider(TorrentProvider):
with bs4_parser(data) as html:
torrent_table = html.find('table', class_='tl')
torrent_rows = torrent_table.find_all('tr')
if not torrent_table:
sickrage.app.log.debug('Data returned from provider does not contain any torrents')
return results
# Continue only if at least one Release is found
torrent_rows = torrent_table.find_all('tr')
if len(torrent_rows) < 2:
sickrage.app.log.debug('Data returned from provider does not contain any torrents')
return results
......
......@@ -82,7 +82,6 @@ class LimeTorrentsProvider(TorrentProvider):
with bs4_parser(data) as html:
torrent_table = html.find('table', class_='table2')
if not torrent_table:
sickrage.app.log.debug('Data returned from provider does not contain any torrents')
return results
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment