added auto-backup feature for app data

.gitlab-ci.yml

@@ -16,8 +16,8 @@ stages:
 # NODE_ENV: "development"
 # script:
 # - apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev
-# - yarn install --pure-lockfile --cache-folder .yarn-cache
-# - yarn run build
+# - npm install
+# - npm run build
 # only:
 # - [email protected]/sickrage
 # cache:
@@ -176,8 +176,8 @@ build_master:
  - bumpversion --allow-dirty release package.json sickrage/version.txt sickrage/__init__.py
  - RELEASE_VERSION=$(awk -F '"' '/^__version__/ {print $2}' sickrage/__init__.py)
  - npx auto-changelog -v $RELEASE_VERSION --hide-credit --package --commit-limit false --ignore-commit-pattern \[TASK\].*
- - yarn install --pure-lockfile --cache-folder .yarn-cache
- - yarn run build
+ - npm install
+ - npm run build
  - python checksum-generator.py
  - git checkout -b release/$RELEASE_VERSION
  - git fetch --all
@@ -218,14 +218,14 @@ build_develop:
  - export PATH="$CARGO_HOME/bin:$PATH"
  - apk add --no-cache git gcc libffi-dev python3-dev musl-dev openssl-dev curl
  - curl https://sh.rustup.rs -sSf | sh -s -- -y
- - yarn install --pure-lockfile --cache-folder .yarn-cache
+ - npm install
  - pip install -U pip
  - pip install bumpversion
  - pip install -r requirements-dev.txt
  - bumpversion --allow-dirty dev package.json sickrage/version.txt sickrage/__init__.py
  - RELEASE_VERSION=$(awk -F '"' '/^__version__/ {print $2}' sickrage/__init__.py)
  - npx auto-changelog -v $RELEASE_VERSION --hide-credit --unreleased --package --commit-limit false --ignore-commit-pattern \[TASK\].*
- - yarn run build
+ - npm run build
  - python checksum-generator.py
  - python setup.py extract_messages
  - python setup.py init_catalog -l en_US

sickrage/core/__init__.py

@@ -51,6 +51,7 @@ from sickrage.core.amqp.consumer import AMQPConsumer
 from sickrage.core.announcements import Announcements
 from sickrage.core.api import API
 from sickrage.core.auth import AuthServer
+from sickrage.core.auto_backup import AutoBackup
 from sickrage.core.common import Quality, Qualities, EpisodeStatus
 from sickrage.core.config import Config
 from sickrage.core.config.helpers import change_gui_lang
@@ -227,6 +228,7 @@ class Core(object):
  self.subtitle_searcher = None
  self.auto_postprocessor = None
  self.upnp_client = None
+ self.auto_backup = None
  self.auth_server = None
  self.announcements = None
  self.api = None
@@ -274,6 +276,7 @@ class Core(object):
  self.subtitle_searcher = SubtitleSearcher()
  self.auto_postprocessor = AutoPostProcessor()
  self.upnp_client = UPNPClient()
+ self.auto_backup = AutoBackup()
  self.announcements = Announcements()
  self.amqp_consumer = AMQPConsumer()
 
@@ -527,6 +530,17 @@ class Core(object):
  id=self.upnp_client.name
  )
 
+ # add auto backup job
+ self.scheduler.add_job(
+ self.auto_backup.task,
+ IntervalTrigger(
+ hours=self.config.general.auto_backup_freq,
+ timezone='utc'
+ ),
+ name=self.auto_backup.name,
+ id=self.auto_backup.name
+ )
+
  # start queues
  self.search_queue.start_worker(self.config.general.max_queue_workers)
  self.show_queue.start_worker(self.config.general.max_queue_workers)

sickrage/core/auto_backup.py

+import sickrage
+from sickrage.core.helpers import backup_app_data
+
+
+class AutoBackup(object):
+ def __init__(self, *args, **kwargs):
+ self.name = "AUTO-BACKUP"
+
+ def task(self):
+ if not sickrage.app.config.general.auto_backup_enable:
+ return
+
+ sickrage.app.log.info("Performing automatic backup of SiCKRAGE")
+ backup_app_data(sickrage.app.config.general.auto_backup_dir, backup_type="auto", keep_num=sickrage.app.config.general.auto_backup_keep_num)
+ sickrage.app.log.info("Finished automatic backup of SiCKRAGE")
\ No newline at end of file

sickrage/core/config/helpers.py

@@ -187,6 +187,7 @@ def change_unrar_tool(unrar_tool):
  sickrage.app.log.info('Disabling UNPACK setting because no unrar is installed.')
  sickrage.app.config.general.unpack = False
 
+
 def change_nzb_dir(nzb_dir):
  """
  Change NZB blackhole directory
@@ -350,3 +351,16 @@ def change_web_external_port(web_external_port):
  sickrage.app.upnp_client.delete_nat_portmap()
  sickrage.app.config.general.web_external_port = int(web_external_port)
  sickrage.app.upnp_client.add_nat_portmap()
+
+
+def change_auto_backup_freq(freq):
+ """
+ Change frequency of auto-backup thread
+
+ :param freq: New frequency
+ """
+ sickrage.app.config.general.auto_backup_freq = int(freq)
+ if sickrage.app.config.general.auto_backup_freq < 1:
+ sickrage.app.config.general.auto_backup_freq = 1
+
+ sickrage.app.scheduler.reschedule_job(sickrage.app.auto_backup.name, trigger='interval', hours=sickrage.app.config.general.auto_backup_freq)

sickrage/core/databases/cache/migrations/versions/005_Add_Announcements_Table.py

@@ -19,7 +19,7 @@ def upgrade():
  'announcements',
  sa.Column('id', sa.Integer, primary_key=True),
  sa.Column('hash', sa.String(255), unique=True, nullable=False),
- sa.Column('seen', sa.Boolean, server_default='false')
+ sa.Column('seen', sa.Boolean)
  )
 
 

sickrage/core/databases/config/__init__.py

@@ -264,6 +264,10 @@ class ConfigDB(SRDatabase):
  strip_special_file_bits = Column(Boolean, default=True)
  max_queue_workers = Column(Integer, default=5)
  update_video_metadata = Column(Boolean, default=True)
+ auto_backup_enable = Column(Boolean, default=False)
+ auto_backup_freq = Column(Integer, default=24)
+ auto_backup_keep_num = Column(Integer, default=1)
+ auto_backup_dir = Column(Text, default='')
 
  class GUI(base):
  __tablename__ = 'gui'

sickrage/core/databases/config/migrations/versions/009_Add_AutoBackup_Columns_To_General_Table.py

+"""Initial migration
+
+Revision ID: 9
+Revises:
+Create Date: 2017-12-29 14:39:27.854291
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+
+revision = '9'
+down_revision = '8'
+
+
+def upgrade():
+ op.add_column('general', sa.Column('auto_backup_enable', sa.Boolean))
+ op.add_column('general', sa.Column('auto_backup_freq', sa.Integer, server_default='24'))
+ op.add_column('general', sa.Column('auto_backup_keep_num', sa.Integer, server_default='1'))
+ op.add_column('general', sa.Column('auto_backup_dir', sa.Text, server_default=''))
+
+
+def downgrade():
+ pass

sickrage/core/databases/main/migrations/versions/013_Add_Scene_Column_To_TVShow_Table.py

@@ -35,7 +35,7 @@ def upgrade():
  tv_shows = sa.Table('tv_shows', meta, autoload=True)
 
  if not hasattr(tv_shows.c, 'scene'):
- op.add_column('tv_shows', sa.Column('scene', sa.Boolean, server_default='false'))
+ op.add_column('tv_shows', sa.Column('scene', sa.Boolean))
 
  with op.get_context().begin_transaction():
  for row in conn.execute(tv_shows.select()):

sickrage/core/helpers/__init__.py

@@ -915,29 +915,15 @@ def restore_config_zip(archive, target_dir, restore_main_database=True, restore_
  shutil.rmtree(target_dir)
 
 
-def backup_app_data(backup_dir, backup_main_db=True, backup_config_db=True, backup_cache_db=True, backup_image_cache=True, keep_latest=False):
+def backup_app_data(backup_dir, backup_type='manual', backup_main_db=True, backup_config_db=True, backup_cache_db=True, backup_image_cache=True, keep_num=0):
  source = []
 
- # files_list = [
- # 'privatekey.pem',
- # os.path.basename(sickrage.app.config_file)
- # ]
-
- def _keep_latest_backup():
- for x in sorted(glob.glob(os.path.join(backup_dir, '*.zip')), key=os.path.getctime, reverse=True)[1:]:
- os.remove(x)
-
  if not os.path.exists(backup_dir):
  os.mkdir(backup_dir)
 
- if keep_latest:
- _keep_latest_backup()
-
- # individual files
- # for f in files_list:
- # fp = os.path.join(sickrage.app.data_dir, f)
- # if os.path.exists(fp):
- # source += [fp]
+ if keep_num > 0:
+ for x in sorted(glob.glob(os.path.join(backup_dir, f'*{backup_type}*.zip')), key=os.path.getctime, reverse=True)[keep_num:]:
+ os.remove(x)
 
  # databases
  if backup_main_db:
@@ -966,7 +952,7 @@ def backup_app_data(backup_dir, backup_main_db=True, backup_config_db=True, back
  source += [os.path.join(path, filename)]
 
  # ZIP filename
- target = os.path.join(backup_dir, f'sickrage-{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.zip')
+ target = os.path.join(backup_dir, f'sickrage-{backup_type}-{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.zip')
 
  return create_zipfile(source, target, sickrage.app.data_dir)
 

sickrage/core/version_updater.py

@@ -118,7 +118,7 @@ class VersionUpdater(object):
  if not os.path.isdir(backupDir):
  os.mkdir(backupDir)
 
- if backup_app_data(backupDir, keep_latest=True):
+ if backup_app_data(backupDir, keep_num=1):
  sickrage.app.log.info("Config backup successful, updating...")
  sickrage.app.alerts.message(_('Updater'), _('Config backup successful, updating...'))
  return True

sickrage/core/webserver/handlers/config/backup_restore.py

@@ -24,6 +24,7 @@ import os
 from tornado.web import authenticated
 
 import sickrage
+from sickrage.core.config.helpers import change_auto_backup_freq
 from sickrage.core.helpers import backup_app_data, checkbox_to_value, restore_config_zip
 from sickrage.core.webserver import ConfigWebHandler
 from sickrage.core.webserver.handlers.base import BaseHandler
@@ -97,4 +98,25 @@ class ConfigRestoreHandler(BaseHandler):
 class SaveBackupRestoreHandler(BaseHandler):
  @authenticated
  def post(self, *args, **kwargs):
+ backup_dir = self.get_argument('backupDir')
+ auto_backup_enable = self.get_argument('auto_backup_enable', False)
+ auto_backup_freq = self.get_argument('auto_backup_freq')
+ auto_backup_keep_num = self.get_argument('auto_backup_keep_num')
+
+ results = []
+
+ sickrage.app.config.general.auto_backup_dir = backup_dir
+ sickrage.app.config.general.auto_backup_enable = checkbox_to_value(auto_backup_enable)
+ sickrage.app.config.general.auto_backup_keep_num = int(auto_backup_keep_num)
+
+ change_auto_backup_freq(auto_backup_freq)
+
+ sickrage.app.config.save()
+
+ if len(results) > 0:
+ [sickrage.app.log.error(x) for x in results]
+ sickrage.app.alerts.error(_('Error(s) Saving Configuration'), '<br>\n'.join(results))
+ else:
+ sickrage.app.alerts.message(_('[BACKUP] Configuration Saved to Database'))
+
  return self.redirect("/config/backuprestore/")

sickrage/core/webserver/views/config/backup_restore.mako

 <%inherit file="../layouts/config.mako"/>
 <%def name='formaction()'><% return 'saveBackupRestore' %></%def>
+<%!
+ import sickrage
+%>
 <%block name="menus">
  <li class="nav-item px-1"><a class="nav-link" data-toggle="tab" href="#backup">${_('Backup')}</a></li>
  <li class="nav-item px-1"><a class="nav-link" data-toggle="tab" href="#restore">${_('Restore')}</a></li>
@@ -16,34 +19,81 @@
 
  <fieldset class="col-lg-9 col-md-8 col-sm-8 card-text">
  <div class="form-row form-group">
- <div class="col-md-12 component-desc">
- <div class="form-row">
- <div class="col-md-12">
- <div class="input-group">
- <input name="backupDir" id="backupDir" class="form-control"
- placeholder="${_('Select the folder you wish to save your backup file to')}"
- autocapitalize="off"/>
- <div class="input-group-append">
- <span class="input-group-text">
- <a href="#" class="fas fa-download" title="${_('Backup')}" id="Backup"></a>
- </span>
- </div>
- </div>
+ <div class="col-lg-3 col-md-4 col-sm-5">
+ <label class="component-title">${_('Backup folder')}</label>
+ </div>
+ <div class="col-lg-9 col-md-8 col-sm-7 component-desc">
+ <div class="input-group">
+ <input name="backupDir" id="backupDir" class="form-control" value="${sickrage.app.config.general.auto_backup_dir}"
+ placeholder="${_('Select the folder you wish to save your backup file to')}"
+ autocapitalize="off"/>
+ <div class="input-group-append">
+ <span class="input-group-text">
+ <a href="#" class="fas fa-download" title="${_('Manual Backup')}" id="Backup"></a>
+ </span>
  </div>
  </div>
+ </div>
+ </div>
 
- <br/>
+ <div class="form-row form-group">
+ <div class="col-lg-3 col-md-4 col-sm-5">
+ <label class="component-title">${_('Automatic backup')}</label>
+ </div>
+ <div class="col-lg-9 col-md-8 col-sm-7 component-desc">
+ <label for="auto_backup_enable">
+ <input type="checkbox" class="toggle color-primary is-material" name="auto_backup_enable" id="auto_backup_enable" ${('', 'checked')[bool(sickrage.app.config.general.auto_backup_enable)]}/>
+ ${_('Enable Automatic Backups')}
+ </label>
+ </div>
+ </div>
 
- <div class="form-row">
- <div class="col-md-12">
- <label for="auto_backup">
- <input type="checkbox" class="toggle color-primary is-material" name="auto_backup" id="auto_backup" checked/>
- ${_('Automatic Backups')}
- </label>
+ <div class="form-row form-group">
+ <div class="col-lg-3 col-md-4 col-sm-5">
+ <label class="component-title">${_('Automatic backup frequency')}</label>
+ </div>
+ <div class="col-lg-9 col-md-8 col-sm-7 component-desc">
+ <div class="input-group">
+ <div class="input-group-prepend">
+ <span class="input-group-text">
+ <span class="fas fa-clock"></span>
+ </span>
+ </div>
+ <input name="auto_backup_freq"
+ id="auto_backup_freq"
+ value="${sickrage.app.config.general.auto_backup_freq}"
+ placeholder="${_('default = 24')}"
+ title="minimum allowed time is 1 hour"
+ class="form-control"/>
+ <div class="input-group-append">
+ <span class="input-group-text">
+ hour
+ </span>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="form-row form-group">
+ <div class="col-lg-3 col-md-4 col-sm-5">
+ <label class="component-title">${_('Automatic backups to keep')}</label>
+ </div>
+ <div class="col-lg-9 col-md-8 col-sm-7 component-desc">
+ <div class="input-group">
+ <div class="input-group-prepend">
+ <span class="input-group-text">
+ <span class="fas fa-file"></span>
+ </span>
  </div>
+ <input name="auto_backup_keep_num" id="auto_backup_keep_num"
+ value="${sickrage.app.config.general.auto_backup_keep_num}"
+ placeholder="${_('default = 1')}"
+ title="number of backups to keep"
+ class="form-control"/>
  </div>
  </div>
  </div>
+
  <div class="form-row">
  <div class="col-md-12">
  <div class="Backup" id="Backup-result"></div>

sickrage/core/webserver/views/config/general.mako

@@ -159,7 +159,6 @@ c<%inherit file="../layouts/config.mako"/>
  </div>
 
  <div class="form-row form-group">
-
  <div class="col-lg-3 col-md-4 col-sm-5">
  <label class="component-title">${_('Number of Log files saved')}</label>
  </div>
@@ -177,7 +176,6 @@ c<%inherit file="../layouts/config.mako"/>
  class="form-control"/>
  </div>
  </div>
-
  </div>
 
  <div class="form-row form-group">