Commit 96cdca42 authored by echel0n's avatar echel0n
Browse files

Refactored database backup/restore code

Bumped main database to v16
Fixed issue with returning scene absolute number by lookup of indexer absolute number when absolute number is zero and returns multiple rows
Removed psutil from requirements.txt
Refactored file cleanup on start to skip *.pyc files
Remover memory_usage function from helpers
parent 23cfc944
......@@ -51,6 +51,5 @@ certifi
pyasn1
lxml
ipaddress
psutil
cffi
cryptography
\ No newline at end of file
......@@ -220,7 +220,11 @@ def file_cleanup(remove=False):
for root, dirs, files in os.walk(PROG_DIR):
for file in files:
full_filename = pathlib.Path(root).joinpath(file)
if full_filename != pathlib.Path(CHECKSUM_FILE) and full_filename not in valid_files and PROG_DIR in str(full_filename):
if full_filename == pathlib.Path(CHECKSUM_FILE) or full_filename.suffix == '.pyc':
continue
if full_filename not in valid_files and PROG_DIR in str(full_filename):
try:
if remove:
print('Found unwanted file {}, removed!'.format(full_filename))
......
......@@ -226,8 +226,8 @@ class Core(object):
success = restore_app_data(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" % ("FAILED", "SUCCESSFUL")[success])
if success:
self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
# self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
# self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
......
......@@ -28,10 +28,13 @@ from time import sleep
import sqlalchemy
from migrate import DatabaseAlreadyControlledError, DatabaseNotControlledError
from migrate.versioning import api
from sqlalchemy import create_engine, event, inspect
from sqlalchemy import create_engine, event, inspect, MetaData
from sqlalchemy.engine import Engine
from sqlalchemy.exc import OperationalError
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.serializer import loads, dumps
from sqlalchemy.orm import sessionmaker, mapper, scoped_session
from sqlalchemy.util import KeyedTuple
import sickrage
from sickrage.core.helpers import backup_versioned_file
......@@ -160,6 +163,14 @@ class SRDatabase(object):
except DatabaseNotControlledError:
return 0
def get_metadata(self):
return MetaData(bind=self.engine, reflect=True)
def get_base(self):
base = automap_base(metadata=self.get_metadata())
base.prepare()
return base
def integrity_check(self):
if self.db_type == 'sqlite':
if self.session().scalar("PRAGMA integrity_check") != "ok":
......@@ -277,8 +288,24 @@ class SRDatabase(object):
del migrate_tables
del rows
def backup(self):
pass
def restore(self):
pass
def backup(self, filename):
metadata = self.get_metadata()
data = {t: dumps(self.session().query(metadata.tables[t]).all()) for t in metadata.tables}
with open(filename, 'wb') as fh:
pickle.dump(data, fh)
def restore(self, filename):
session = self.session()
metadata = self.get_metadata()
base = self.get_base()
with open(filename, 'rb') as fh:
data_dict = pickle.load(fh)
for table_name, data in data_dict.items():
table = base.classes[table_name]
session.query(table).delete()
for row in loads(data, metadata, session):
if isinstance(row, KeyedTuple):
row = table(**row._asdict())
session.merge(row)
session.commit()
......@@ -34,7 +34,7 @@ class MainDBBase(SRDatabaseBase):
class MainDB(SRDatabase):
def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
super(MainDB, self).__init__('main', 15, db_type, db_prefix, db_host, db_port, db_username, db_password)
super(MainDB, self).__init__('main', 16, db_type, db_prefix, db_host, db_port, db_username, db_password)
MainDBBase.metadata.create_all(self.engine)
for model in MainDBBase._decl_class_registry.values():
if hasattr(model, '__tablename__'):
......
from sqlalchemy import *
from sqlalchemy.exc import NoSuchTableError
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
tv_episodes = Table('tv_episodes', meta, autoload=True)
scene_numbering = Table('scene_numbering', meta, autoload=True)
try:
scene_numbering = Table('scene_numbering', meta, autoload=True)
except NoSuchTableError:
scene_numbering = None
if scene_numbering is not None:
with migrate_engine.begin() as conn:
......
......@@ -23,7 +23,9 @@
import base64
import ctypes
import datetime
import errno
import glob
import gzip
import hashlib
import ipaddress
import os
......@@ -45,7 +47,6 @@ from collections import OrderedDict
from contextlib import contextmanager
from urllib.parse import uses_netloc, urlsplit, urlunsplit, urljoin
import errno
import rarfile
import requests
from bs4 import BeautifulSoup
......@@ -923,10 +924,17 @@ def restore_config_zip(archive, targetDir, restore_database=True, restore_config
with zipfile.ZipFile(archive, 'r', allowZip64=True) as zip_file:
for member in zip_file.namelist():
if not restore_database and member.split('/')[0] in ['main.db', 'main.db-shm', 'main.db-wal', 'cache.db', 'cache.db-shm', 'cache.db-wal']:
if not restore_database and member.split('/')[0] in ['main_db_backup.json',
'main.db', 'main.db-shm',
'main.db-wal',
'cache_db_backup.json',
'cache.db',
'cache.db-shm',
'cache.db-wal']:
continue
if not restore_config and member.split('/')[0] in ['config.ini', 'privatekey.pem']:
if not restore_config and member.split('/')[0] in ['config.ini',
'privatekey.pem']:
continue
if not restore_cache and member.split('/')[0] == 'cache':
......@@ -944,12 +952,6 @@ def backup_app_data(backupDir, keep_latest=False):
source = []
files_list = [
'main.db',
'main.db-shm',
'main.db-wal',
'cache.db',
'cache.db-shm',
'cache.db-wal',
'privatekey.pem',
os.path.basename(sickrage.app.config_file)
]
......@@ -970,6 +972,12 @@ def backup_app_data(backupDir, keep_latest=False):
if os.path.exists(fp):
source += [fp]
# databases
for db in [sickrage.app.main_db, sickrage.app.cache_db]:
backup_file = os.path.join(*[sickrage.app.data_dir, '{}_db_backup.json'.format(db.name)])
db.backup(backup_file)
source += [backup_file]
# cache folder
if sickrage.app.cache_dir:
for (path, dirs, files) in os.walk(sickrage.app.cache_dir, topdown=True):
......@@ -1011,6 +1019,12 @@ def restore_app_data(srcDir, dstDir):
move_file(dstFile, bakFile)
move_file(srcFile, dstFile)
# database
for db in [sickrage.app.main_db, sickrage.app.cache_db]:
backup_file = os.path.join(*[srcDir, '{}_db_backup.json'.format(db.name)])
if os.path.exists(backup_file):
db.restore(backup_file)
# cache
if os.path.exists(os.path.join(srcDir, 'cache')):
if os.path.exists(os.path.join(dstDir, 'cache')):
......@@ -1652,21 +1666,6 @@ def glob_escape(pathname):
return drive + pathname
def memory_usage():
try:
try:
import psutil
p = psutil.Process(sickrage.app.pid)
return pretty_file_size(int(p.memory_info().rss))
except ImportError:
import resource
return pretty_file_size(int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * 1024)
except Exception:
pass
return 'unknown'
def convert_to_timedelta(time_val):
"""
Given a *time_val* (string) such as '5d', returns a `datetime.timedelta`
......
......@@ -316,6 +316,8 @@ def find_scene_absolute_numbering(indexer_id, indexer, absolute_number):
).filter(MainDB.TVEpisode.scene_absolute_number != -1).one()
return dbData.scene_absolute_number
except orm.exc.MultipleResultsFound:
return
except orm.exc.NoResultFound:
return
......
......@@ -5,7 +5,7 @@
from time import time
import sickrage
from sickrage.core.helpers import pretty_file_size, memory_usage
from sickrage.core.helpers import pretty_file_size
%>
<%namespace file="../includes/modals.mako" import="mainModals"/>
......@@ -393,10 +393,6 @@
## ${_('Backlog Search:')} <span
## class="text-primary">${str(sickrage.app.scheduler.get_job('BACKLOG').next_run_time).split('.')[0]}</span>
## |
## ${_('Memory used:')}
## <span class="text-primary">
## ${memory_usage()}
## </span> |
## ${_('Load time:')}
## <span class="text-primary">
## ${"{:10.4f}".format(time() - srStartTime)}s
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment