Commit 2c41b481 authored by echel0n's avatar echel0n

Merge branch 'refactor-database-migrations' into develop

# Conflicts:
#	sickrage/core/databases/main/db_repository/versions/016_Merge_Scene_Numbering_Table_With_TVEpisodes_Table.py
parents a93d6785 cd2ef1b3
......@@ -42,7 +42,7 @@ simplejson==3.17.2
service_identity==18.1.0
knowit==0.2.4
sqlalchemy==1.3.18
sqlalchemy-migrate==0.13.0
alembic==1.4.2
mutagen==1.45.1
deluge-client==1.9.0
dirsync==2.2.5
......
......@@ -223,6 +223,7 @@ class Core(object):
# check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
self.log.info('Performing restore of backup files')
success = restore_app_data(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" % ("FAILED", "SUCCESSFUL")[success])
if success:
......@@ -274,9 +275,9 @@ class Core(object):
self.log.info("Performing migrations on {} database".format(db.name))
db.migrate()
# sync database repo
self.log.info("Performing sync on {} database".format(db.name))
db.sync_db_repo()
# upgrade database
self.log.info("Performing upgrades on {} database".format(db.name))
db.upgrade()
# cleanup
self.log.info("Performing cleanup on {} database".format(db.name))
......
......@@ -25,19 +25,22 @@ import threading
from collections import OrderedDict
from time import sleep
import alembic.command
import alembic.config
import alembic.script
import sqlalchemy
from migrate import DatabaseAlreadyControlledError, DatabaseNotControlledError
from migrate.versioning import api
from sqlalchemy import create_engine, event, inspect, MetaData
from sqlalchemy.engine import Engine
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from sqlalchemy import create_engine, event, inspect, MetaData, Index
from sqlalchemy.engine import Engine, reflection
from sqlalchemy.exc import OperationalError
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.serializer import loads, dumps
from sqlalchemy.orm import sessionmaker, mapper, scoped_session
from sqlalchemy.sql.ddl import CreateTable, CreateIndex
from sqlalchemy.util import KeyedTuple
import sickrage
from sickrage.core.helpers import backup_versioned_file
@event.listens_for(Engine, "connect")
......@@ -119,10 +122,8 @@ class SRDatabaseBase(object):
class SRDatabase(object):
def __init__(self, name, db_version=0, db_type='sqlite', db_prefix='sickrage', db_host='localhost', db_port='3306', db_username='sickrage',
db_password='sickrage'):
def __init__(self, name, db_type='sqlite', db_prefix='sickrage', db_host='localhost', db_port='3306', db_username='sickrage', db_password='sickrage'):
self.name = name
self.db_version = db_version
self.db_type = db_type
self.db_prefix = db_prefix
self.db_host = db_host
......@@ -133,17 +134,17 @@ class SRDatabase(object):
self.tables = {}
self.db_path = os.path.join(sickrage.app.data_dir, '{}.db'.format(self.name))
self.db_repository = os.path.join(os.path.dirname(__file__), self.name, 'db_repository')
self.db_migrations_path = os.path.join(os.path.dirname(__file__), self.name, 'migrations')
self.session = scoped_session(sessionmaker(class_=ContextSession, bind=self.engine))
if not self.version:
api.version_control(self.engine, self.db_repository, api.version(self.db_repository))
else:
try:
api.version_control(self.engine, self.db_repository)
except DatabaseAlreadyControlledError:
pass
if self.engine.dialect.has_table(self.engine, 'migrate_version'):
migrate_version = self.engine.execute("select version from migrate_version").fetchone().version
alembic.command.stamp(self.get_alembic_config(), str(migrate_version))
self.engine.execute("drop table migrate_version")
if not self.engine.dialect.has_table(self.engine, 'alembic_version'):
alembic.command.stamp(self.get_alembic_config(), 'head')
@property
def engine(self):
......@@ -158,10 +159,29 @@ class SRDatabase(object):
@property
def version(self):
try:
return int(api.db_version(self.engine, self.db_repository))
except DatabaseNotControlledError:
return 0
context = MigrationContext.configure(self.engine)
current_rev = context.get_current_revision()
return current_rev
def upgrade(self):
db_version = int(self.version)
alembic_version = int(ScriptDirectory.from_config(self.get_alembic_config()).get_current_head())
backup_filename = os.path.join(sickrage.app.data_dir, '{}_db_backup_{}.json'.format(self.name, datetime.datetime.now().strftime('%Y%m%d_%H%M%S')))
if db_version < alembic_version:
sickrage.app.log.info('Upgrading {} database to v{}'.format(self.name, alembic_version))
self.backup(backup_filename)
alembic.command.upgrade(self.get_alembic_config(), 'head')
def get_alembic_config(self):
config = alembic.config.Config()
config.set_main_option('script_location', self.db_migrations_path)
config.set_main_option('sqlalchemy.url', str(self.engine.url))
config.set_main_option('url', str(self.engine.url))
return config
def get_metadata(self):
return MetaData(bind=self.engine, reflect=True)
......@@ -177,22 +197,6 @@ class SRDatabase(object):
sickrage.app.log.fatal("{} database file {} is damaged, please restore a backup"
" or delete the database file and restart SiCKRAGE".format(self.name.capitalize(), self.db_path))
def sync_db_repo(self):
if self.version < self.db_version:
if self.db_type == 'sqlite':
backup_versioned_file(self.db_path, self.version)
backup_versioned_file(self.db_path + '-shm', self.version)
backup_versioned_file(self.db_path + '-wal', self.version)
api.upgrade(self.engine, self.db_repository)
sickrage.app.log.info('Upgraded {} database to version {}'.format(self.name, self.version))
elif self.version > self.db_version:
if self.db_type == 'sqlite':
backup_versioned_file(self.db_path, self.version)
backup_versioned_file(self.db_path + '-shm', self.version)
backup_versioned_file(self.db_path + '-wal', self.version)
api.downgrade(self.engine, self.db_repository, self.db_version)
sickrage.app.log.info('Downgraded {} database to version {}'.format(self.name, self.version))
def cleanup(self):
pass
......@@ -289,23 +293,57 @@ class SRDatabase(object):
del rows
def backup(self, filename):
metadata = self.get_metadata()
data = {t: dumps(self.session().query(metadata.tables[t]).all(), protocol=pickle.DEFAULT_PROTOCOL) for t in metadata.tables}
meta = self.get_metadata()
backup_dict = {
'schema': {},
'indexes': {},
'data': {}
}
for table_name, table_object in meta.tables.items():
backup_dict['indexes'].update({table_name: []})
backup_dict['schema'].update({table_name: str(CreateTable(table_object))})
backup_dict['data'].update({table_name: dumps(self.session().query(table_object).all(), protocol=pickle.DEFAULT_PROTOCOL)})
for index in reflection.Inspector.from_engine(self.engine).get_indexes(table_name):
cols = [table_object.c[col] for col in index['column_names']]
idx = Index(index['name'], *cols)
backup_dict['indexes'][table_name].append(str(CreateIndex(idx)))
with open(filename, 'wb') as fh:
pickle.dump(data, fh, protocol=pickle.DEFAULT_PROTOCOL)
pickle.dump(backup_dict, fh, protocol=pickle.DEFAULT_PROTOCOL)
def restore(self, filename):
session = self.session()
metadata = self.get_metadata()
base = self.get_base()
with open(filename, 'rb') as fh:
data_dict = pickle.load(fh)
for table_name, data in data_dict.items():
table = base.classes[table_name]
session.query(table).delete()
for row in loads(data, metadata, session):
if isinstance(row, KeyedTuple):
row = table(**row._asdict())
session.merge(row)
session.commit()
backup_dict = pickle.load(fh)
# drop all tables
self.get_base().metadata.drop_all()
# restore schema
if backup_dict.get('schema', None):
for table_name, schema in backup_dict['schema'].items():
session.execute(schema)
session.commit()
# restore indexes
if backup_dict.get('indexes', None):
for table_name, indexes in backup_dict['indexes'].items():
for index in indexes:
session.execute(index)
session.commit()
# restore data
if backup_dict.get('data', None):
base = self.get_base()
meta = self.get_metadata()
for table_name, data in backup_dict['data'].items():
table = base.classes[table_name]
session.query(table).delete()
for row in loads(data, meta, session):
if isinstance(row, KeyedTuple):
row = table(**row._asdict())
session.merge(row)
session.commit()
......@@ -29,7 +29,7 @@ class CacheDBBase(SRDatabaseBase):
class CacheDB(SRDatabase):
def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
super(CacheDB, self).__init__('cache', 8, db_type, db_prefix, db_host, db_port, db_username, db_password)
super(CacheDB, self).__init__('cache', db_type, db_prefix, db_host, db_port, db_username, db_password)
CacheDBBase.metadata.create_all(self.engine)
for model in CacheDBBase._decl_class_registry.values():
if hasattr(model, '__tablename__'):
......
This is a database migration repository.
More information at
http://code.google.com/p/sqlalchemy-migrate/
#!/usr/bin/env python
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=SiCKRAGE
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False
from sqlalchemy import *
from migrate import *
def upgrade(migrate_engine):
# Upgrade operations go here. Don't create your own engine; bind
# migrate_engine to your metadata
pass
def downgrade(migrate_engine):
# Operations to reverse the above upgrade go here.
pass
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from migrate.changeset.constraint import PrimaryKeyConstraint
from sqlalchemy import MetaData, Table, String
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
last_search = Table('last_search', meta, autoload=True)
with migrate_engine.begin() as conn:
conn.execute(last_search.delete())
last_search.c.provider.alter(type=String(32))
primary_key = PrimaryKeyConstraint(last_search.c.provider)
primary_key.create()
last_search.c.id.drop()
def downgrade(migrate_engine):
pass
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from migrate.changeset.constraint import PrimaryKeyConstraint
from sqlalchemy import MetaData, Table, String
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
providers = Table('providers', meta, autoload=True)
if hasattr(providers.c, 'indexer_id'):
providers.c.indexer_id.alter(name='series_id')
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
providers = Table('providers', meta, autoload=True)
if hasattr(providers.c, 'series_id'):
providers.c.series_id.alter(name='indexer_id')
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
import json
import os
from json import JSONDecodeError
from sqlalchemy import *
import sickrage
def upgrade(migrate_engine):
pass
def downgrade(migrate_engine):
pass
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from sqlalchemy import *
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
oauth2_token = Table('oauth2_token', meta, autoload=True)
if not hasattr(oauth2_token.c, 'session_state'):
session_state = Column('session_state', Text, default='')
session_state.create(oauth2_token)
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
oauth2_token = Table('oauth2_token', meta, autoload=True)
if hasattr(oauth2_token.c, 'session_state'):
oauth2_token.c.session_state.drop()
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from sqlalchemy import *
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
oauth2_token = Table('oauth2_token', meta, autoload=True)
if not hasattr(oauth2_token.c, 'token_type'):
token_type = Column('token_type', Text, default='bearer')
token_type.create(oauth2_token)
def downgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
oauth2_token = Table('oauth2_token', meta, autoload=True)
if hasattr(oauth2_token.c, 'token_type'):
oauth2_token.c.token_type.drop()
# ##############################################################################
# Author: echel0n <[email protected]>
# URL: https://sickrage.ca/
# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
# -
# This file is part of SiCKRAGE.
# -
# SiCKRAGE is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# -
# SiCKRAGE is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# -
# You should have received a copy of the GNU General Public License
# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
# ##############################################################################
from sqlalchemy import *
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
quicksearch_shows = Table('quicksearch_shows', meta, autoload=True)
if quicksearch_shows is not None:
quicksearch_shows.drop()
quicksearch_episodes = Table('quicksearch_episodes', meta, autoload=True)
if quicksearch_episodes is not None:
quicksearch_episodes.drop()
def downgrade(migrate_engine):
pass
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
# fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}
"""Initial migration
Revision ID: 1
Revises:
Create Date: 2017-12-29 14:39:27.854291
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '1'
down_revision = None
def upgrade():
pass
def downgrade():
pass
"""Initial migration
Revision ID: 2
Revises:
Create Date: 2017-12-29 14:39:27.854291
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '2'