Refactoring database migrations to use Alembic

requirements.txt

@@ -42,7 +42,7 @@ simplejson==3.17.2
 service_identity==18.1.0
 knowit==0.2.4
 sqlalchemy==1.3.18
-sqlalchemy-migrate==0.13.0
+alembic==1.4.2
 mutagen==1.45.1
 deluge-client==1.9.0
 dirsync==2.2.5

sickrage/core/__init__.py

@@ -223,6 +223,7 @@ class Core(object):
 
  # check if we need to perform a restore first
  if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
+ self.log.info('Performing restore of backup files')
  success = restore_app_data(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
  self.log.info("Restoring SiCKRAGE backup: %s!" % ("FAILED", "SUCCESSFUL")[success])
  if success:
@@ -274,9 +275,9 @@ class Core(object):
  self.log.info("Performing migrations on {} database".format(db.name))
  db.migrate()
 
- # sync database repo
- self.log.info("Performing sync on {} database".format(db.name))
- db.sync_db_repo()
+ # upgrade database
+ self.log.info("Performing upgrades on {} database".format(db.name))
+ db.upgrade()
 
  # cleanup
  self.log.info("Performing cleanup on {} database".format(db.name))

sickrage/core/databases/__init__.py

@@ -25,19 +25,22 @@ import threading
 from collections import OrderedDict
 from time import sleep
 
+import alembic.command
+import alembic.config
+import alembic.script
 import sqlalchemy
-from migrate import DatabaseAlreadyControlledError, DatabaseNotControlledError
-from migrate.versioning import api
-from sqlalchemy import create_engine, event, inspect, MetaData
-from sqlalchemy.engine import Engine
+from alembic.runtime.migration import MigrationContext
+from alembic.script import ScriptDirectory
+from sqlalchemy import create_engine, event, inspect, MetaData, Index
+from sqlalchemy.engine import Engine, reflection
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.ext.serializer import loads, dumps
 from sqlalchemy.orm import sessionmaker, mapper, scoped_session
+from sqlalchemy.sql.ddl import CreateTable, CreateIndex
 from sqlalchemy.util import KeyedTuple
 
 import sickrage
-from sickrage.core.helpers import backup_versioned_file
 
 
 @event.listens_for(Engine, "connect")
@@ -119,10 +122,8 @@ class SRDatabaseBase(object):
 
 
 class SRDatabase(object):
- def __init__(self, name, db_version=0, db_type='sqlite', db_prefix='sickrage', db_host='localhost', db_port='3306', db_username='sickrage',
- db_password='sickrage'):
+ def __init__(self, name, db_type='sqlite', db_prefix='sickrage', db_host='localhost', db_port='3306', db_username='sickrage', db_password='sickrage'):
  self.name = name
- self.db_version = db_version
  self.db_type = db_type
  self.db_prefix = db_prefix
  self.db_host = db_host
@@ -133,17 +134,17 @@ class SRDatabase(object):
  self.tables = {}
 
  self.db_path = os.path.join(sickrage.app.data_dir, '{}.db'.format(self.name))
- self.db_repository = os.path.join(os.path.dirname(__file__), self.name, 'db_repository')
+ self.db_migrations_path = os.path.join(os.path.dirname(__file__), self.name, 'migrations')
 
  self.session = scoped_session(sessionmaker(class_=ContextSession, bind=self.engine))
 
- if not self.version:
- api.version_control(self.engine, self.db_repository, api.version(self.db_repository))
- else:
- try:
- api.version_control(self.engine, self.db_repository)
-  except DatabaseAlreadyControlledError:
-  pass
+ if self.engine.dialect.has_table(self.engine, 'migrate_version'):
+ migrate_version = self.engine.execute("select version from migrate_version").fetchone().version
+  alembic.command.stamp(self.get_alembic_config(), str(migrate_version))
+ self.engine.execute("drop table migrate_version")
+
+ if not self.engine.dialect.has_table(self.engine, 'alembic_version'):
+ alembic.command.stamp(self.get_alembic_config(), 'head')
 
  @property
  def engine(self):
@@ -158,10 +159,29 @@ class SRDatabase(object):
 
  @property
  def version(self):
- try:
- return int(api.db_version(self.engine, self.db_repository))
- except DatabaseNotControlledError:
- return 0
+ context = MigrationContext.configure(self.engine)
+ current_rev = context.get_current_revision()
+ return current_rev
+
+ def upgrade(self):
+ db_version = int(self.version)
+ alembic_version = int(ScriptDirectory.from_config(self.get_alembic_config()).get_current_head())
+
+ backup_filename = os.path.join(sickrage.app.data_dir, '{}_db_backup_{}.json'.format(self.name, datetime.datetime.now().strftime('%Y%m%d_%H%M%S')))
+
+ if db_version < alembic_version:
+ sickrage.app.log.info('Upgrading {} database to v{}'.format(self.name, alembic_version))
+
+ self.backup(backup_filename)
+
+ alembic.command.upgrade(self.get_alembic_config(), 'head')
+
+ def get_alembic_config(self):
+ config = alembic.config.Config()
+ config.set_main_option('script_location', self.db_migrations_path)
+ config.set_main_option('sqlalchemy.url', str(self.engine.url))
+ config.set_main_option('url', str(self.engine.url))
+ return config
 
  def get_metadata(self):
  return MetaData(bind=self.engine, reflect=True)
@@ -177,22 +197,6 @@ class SRDatabase(object):
  sickrage.app.log.fatal("{} database file {} is damaged, please restore a backup"
  " or delete the database file and restart SiCKRAGE".format(self.name.capitalize(), self.db_path))
 
- def sync_db_repo(self):
- if self.version < self.db_version:
- if self.db_type == 'sqlite':
- backup_versioned_file(self.db_path, self.version)
- backup_versioned_file(self.db_path + '-shm', self.version)
- backup_versioned_file(self.db_path + '-wal', self.version)
- api.upgrade(self.engine, self.db_repository)
- sickrage.app.log.info('Upgraded {} database to version {}'.format(self.name, self.version))
- elif self.version > self.db_version:
- if self.db_type == 'sqlite':
- backup_versioned_file(self.db_path, self.version)
- backup_versioned_file(self.db_path + '-shm', self.version)
- backup_versioned_file(self.db_path + '-wal', self.version)
- api.downgrade(self.engine, self.db_repository, self.db_version)
- sickrage.app.log.info('Downgraded {} database to version {}'.format(self.name, self.version))
-
  def cleanup(self):
  pass
 
@@ -289,23 +293,57 @@ class SRDatabase(object):
  del rows
 
  def backup(self, filename):
- metadata = self.get_metadata()
- data = {t: dumps(self.session().query(metadata.tables[t]).all(), protocol=pickle.DEFAULT_PROTOCOL) for t in metadata.tables}
+ meta = self.get_metadata()
+
+ backup_dict = {
+ 'schema': {},
+ 'indexes': {},
+ 'data': {}
+ }
+
+ for table_name, table_object in meta.tables.items():
+ backup_dict['indexes'].update({table_name: []})
+ backup_dict['schema'].update({table_name: str(CreateTable(table_object))})
+ backup_dict['data'].update({table_name: dumps(self.session().query(table_object).all(), protocol=pickle.DEFAULT_PROTOCOL)})
+ for index in reflection.Inspector.from_engine(self.engine).get_indexes(table_name):
+ cols = [table_object.c[col] for col in index['column_names']]
+ idx = Index(index['name'], *cols)
+ backup_dict['indexes'][table_name].append(str(CreateIndex(idx)))
+
  with open(filename, 'wb') as fh:
- pickle.dump(data, fh, protocol=pickle.DEFAULT_PROTOCOL)
+ pickle.dump(backup_dict, fh, protocol=pickle.DEFAULT_PROTOCOL)
 
  def restore(self, filename):
  session = self.session()
- metadata = self.get_metadata()
- base = self.get_base()
 
  with open(filename, 'rb') as fh:
- data_dict = pickle.load(fh)
- for table_name, data in data_dict.items():
- table = base.classes[table_name]
- session.query(table).delete()
- for row in loads(data, metadata, session):
- if isinstance(row, KeyedTuple):
- row = table(**row._asdict())
- session.merge(row)
- session.commit()
+ backup_dict = pickle.load(fh)
+
+ # drop all tables
+ self.get_base().metadata.drop_all()
+
+ # restore schema
+ if backup_dict.get('schema', None):
+ for table_name, schema in backup_dict['schema'].items():
+ session.execute(schema)
+ session.commit()
+
+ # restore indexes
+ if backup_dict.get('indexes', None):
+ for table_name, indexes in backup_dict['indexes'].items():
+ for index in indexes:
+ session.execute(index)
+ session.commit()
+
+ # restore data
+ if backup_dict.get('data', None):
+ base = self.get_base()
+ meta = self.get_metadata()
+ for table_name, data in backup_dict['data'].items():
+ table = base.classes[table_name]
+ session.query(table).delete()
+ for row in loads(data, meta, session):
+ if isinstance(row, KeyedTuple):
+ row = table(**row._asdict())
+ session.merge(row)
+ session.commit()

sickrage/core/databases/cache/__init__.py

@@ -29,7 +29,7 @@ class CacheDBBase(SRDatabaseBase):
 
 class CacheDB(SRDatabase):
  def __init__(self, db_type, db_prefix, db_host, db_port, db_username, db_password):
- super(CacheDB, self).__init__('cache', 8, db_type, db_prefix, db_host, db_port, db_username, db_password)
+ super(CacheDB, self).__init__('cache', db_type, db_prefix, db_host, db_port, db_username, db_password)
  CacheDBBase.metadata.create_all(self.engine)
  for model in CacheDBBase._decl_class_registry.values():
  if hasattr(model, '__tablename__'):

sickrage/core/databases/cache/db_repository/README

-This is a database migration repository.
-
-More information at
-http://code.google.com/p/sqlalchemy-migrate/

sickrage/core/databases/cache/db_repository/manage.py

-#!/usr/bin/env python
-from migrate.versioning.shell import main
-
-if __name__ == '__main__':
- main(debug='False')

sickrage/core/databases/cache/db_repository/migrate.cfg

-[db_settings]
-# Used to identify which repository this database is versioned under.
-# You can use the name of your project.
-repository_id=SiCKRAGE
-
-# The name of the database table used to track the schema version.
-# This name shouldn't already be used by your project.
-# If this is changed once a database is under version control, you'll need to
-# change the table name in each database too.
-version_table=migrate_version
-
-# When committing a change script, Migrate will attempt to generate the
-# sql for all supported databases; normally, if one of them fails - probably
-# because you don't have that database installed - it is ignored and the
-# commit continues, perhaps ending successfully.
-# Databases in this list MUST compile successfully during a commit, or the
-# entire commit will fail. List the databases your application will actually
-# be using to ensure your updates to that database work properly.
-# This must be a list; example: ['postgres','sqlite']
-required_dbs=[]
-
-# When creating new change scripts, Migrate will stamp the new script with
-# a version number. By default this is latest_version + 1. You can set this
-# to 'true' to tell Migrate to use the UTC timestamp instead.
-use_timestamp_numbering=False

sickrage/core/databases/cache/db_repository/versions/001_Add_Initial_Tables.py

-from sqlalchemy import *
-from migrate import *
-
-
-def upgrade(migrate_engine):
- # Upgrade operations go here. Don't create your own engine; bind
- # migrate_engine to your metadata
- pass
-
-
-def downgrade(migrate_engine):
- # Operations to reverse the above upgrade go here.
- pass

sickrage/core/databases/cache/db_repository/versions/002_Remove_ID_Column_From_LastSearch_Table.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-
-from migrate.changeset.constraint import PrimaryKeyConstraint
-from sqlalchemy import MetaData, Table, String
-
-
-def upgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- last_search = Table('last_search', meta, autoload=True)
-
- with migrate_engine.begin() as conn:
- conn.execute(last_search.delete())
-
- last_search.c.provider.alter(type=String(32))
- primary_key = PrimaryKeyConstraint(last_search.c.provider)
- primary_key.create()
- last_search.c.id.drop()
-
-
-def downgrade(migrate_engine):
- pass

sickrage/core/databases/cache/db_repository/versions/003_Rename_IndexerID_To_SeriesID_On_Provider_Table.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-
-from migrate.changeset.constraint import PrimaryKeyConstraint
-from sqlalchemy import MetaData, Table, String
-
-
-def upgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- providers = Table('providers', meta, autoload=True)
-
- if hasattr(providers.c, 'indexer_id'):
- providers.c.indexer_id.alter(name='series_id')
-
-
-def downgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- providers = Table('providers', meta, autoload=True)
-
- if hasattr(providers.c, 'series_id'):
- providers.c.series_id.alter(name='indexer_id')

sickrage/core/databases/cache/db_repository/versions/005_Add_Announcements_Table.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-import json
-import os
-from json import JSONDecodeError
-
-from sqlalchemy import *
-
-import sickrage
-
-
-def upgrade(migrate_engine):
- pass
-
-
-def downgrade(migrate_engine):
- pass

sickrage/core/databases/cache/db_repository/versions/006_Add_Session_State_Column_To_OAuth2Token_Table.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-
-from sqlalchemy import *
-
-
-def upgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- oauth2_token = Table('oauth2_token', meta, autoload=True)
- if not hasattr(oauth2_token.c, 'session_state'):
- session_state = Column('session_state', Text, default='')
- session_state.create(oauth2_token)
-
-
-def downgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- oauth2_token = Table('oauth2_token', meta, autoload=True)
- if hasattr(oauth2_token.c, 'session_state'):
- oauth2_token.c.session_state.drop()

sickrage/core/databases/cache/db_repository/versions/007_Add_Token_Type_Column_To_OAuth2Token_Table.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-
-from sqlalchemy import *
-
-
-def upgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- oauth2_token = Table('oauth2_token', meta, autoload=True)
- if not hasattr(oauth2_token.c, 'token_type'):
- token_type = Column('token_type', Text, default='bearer')
- token_type.create(oauth2_token)
-
-
-def downgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- oauth2_token = Table('oauth2_token', meta, autoload=True)
- if hasattr(oauth2_token.c, 'token_type'):
- oauth2_token.c.token_type.drop()

sickrage/core/databases/cache/db_repository/versions/008_Drop_QuickSearch_Tables.py

-# ##############################################################################
-# Author: echel0n <[email protected]>
-# URL: https://sickrage.ca/
-# Git: https://git.sickrage.ca/SiCKRAGE/sickrage.git
-# -
-# This file is part of SiCKRAGE.
-# -
-# SiCKRAGE is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# -
-# SiCKRAGE is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -
-# You should have received a copy of the GNU General Public License
-# along with SiCKRAGE. If not, see <http://www.gnu.org/licenses/>.
-# ##############################################################################
-
-from sqlalchemy import *
-
-
-def upgrade(migrate_engine):
- meta = MetaData(bind=migrate_engine)
- quicksearch_shows = Table('quicksearch_shows', meta, autoload=True)
- if quicksearch_shows is not None:
- quicksearch_shows.drop()
-
- quicksearch_episodes = Table('quicksearch_episodes', meta, autoload=True)
- if quicksearch_episodes is not None:
-  quicksearch_episodes.drop()
-
-
-def downgrade(migrate_engine):
- pass

sickrage/core/databases/cache/migrations/env.py

+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+# fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL
+ and not an Engine, though an Engine is acceptable
+ here as well. By skipping the Engine creation
+ we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the
+ script output.
+
+ """
+ url = config.get_main_option("sqlalchemy.url")
+ context.configure(
+ url=url,
+ target_metadata=target_metadata,
+ literal_binds=True,
+ dialect_opts={"paramstyle": "named"},
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+ connectable = engine_from_config(
+ config.get_section(config.config_ini_section),
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ )
+
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection, target_metadata=target_metadata
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()

sickrage/core/databases/cache/migrations/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}

sickrage/core/databases/cache/migrations/versions/001_Add_Initial_Tables.py

+"""Initial migration
+
+Revision ID: 1
+Revises:
+Create Date: 2017-12-29 14:39:27.854291
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1'
+down_revision = None
+
+
+def upgrade():
+ pass
+
+
+def downgrade():
+ pass

sickrage/core/databases/cache/migrations/versions/002_Remove_ID_Column_From_LastSearch_Table.py

+"""Initial migration
+
+Revision ID: 2
+Revises:
+Create Date: 2017-12-29 14:39:27.854291
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2'