Commit 319186bb authored by echel0n's avatar echel0n

Merge branch 'release/9.4.74'

parents 6ba62e2c 78d6b6ec
# Changelog
- * 24af3a5 - 2019-02-24: Release v9.4.73
- * 6b8c9b2 - 2019-02-24: Release v9.4.74
- * b690738 - 2019-02-24: Fixed issue with default add show options and add show year feature.
- * 90fa33d - 2019-02-24: Pre-Release v9.4.74.dev2
- * b2896ad - 2019-02-24: Updated AniDB code for post-processor. Added get episode function to AniDB helper.
- * 1ebefb4 - 2019-02-24: Pre-Release v9.4.74.dev1
- * c26e5b1 - 2019-02-24: Refactored AniDB code.
- * 41039e5 - 2019-02-24: Fixed KeyError for twitter notifier.
- * 135d868 - 2019-02-24: Fixed AttributeError for gktorrent provider. Refactored log messages for rss cache updater to debug.
- * ffd8fa4 - 2019-02-24: Release v9.4.73
- * 47572b6 - 2019-02-24: Release v9.4.72
- * b86d9d2 - 2019-02-23: Release v9.4.71
- * 342b1de - 2019-02-23: Added ability to save `add_show_year` as a default option when adding new shows.
......
......@@ -39,7 +39,6 @@ from fake_useragent import UserAgent
from keycloak.realm import KeycloakRealm
from tornado.ioloop import IOLoop
import adba
import sickrage
from sickrage.core.api import API
from sickrage.core.caches.name_cache import NameCache
......@@ -296,17 +295,6 @@ class Core(object):
except Exception:
continue
# init anidb connection
if self.config.use_anidb:
def anidb_logger(msg):
return self.log.debug("AniDB: {} ".format(msg))
try:
self.adba_connection = adba.Connection(keepAlive=True, log=anidb_logger)
self.adba_connection.auth(self.config.anidb_username, self.config.anidb_password)
except Exception as e:
self.log.warning("AniDB exception msg: %r " % repr(e))
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
......
......@@ -20,7 +20,6 @@
from __future__ import unicode_literals
import sickrage
from adba.aniDBerrors import AniDBCommandTimeoutError
class BlackAndWhiteList(object):
......@@ -87,7 +86,6 @@ class BlackAndWhiteList(object):
if dbData:
sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
def _load_list(self, table):
"""
DB: Fetch keywords for current show
......@@ -143,32 +141,3 @@ class BlackAndWhiteList(object):
class BlackWhitelistNoShowIDException(Exception):
"""No show_id was given"""
def short_group_names(groups):
"""
Find AniDB short group names for release groups
:param groups: list of groups to find short group names for
:return: list of shortened group names
"""
groups = groups.split(",")
shortGroupList = []
if sickrage.app.adba_connection:
for groupName in groups:
try:
group = sickrage.app.adba_connection.group(gname=groupName)
except AniDBCommandTimeoutError:
sickrage.app.log.debug("Timeout while loading group from AniDB. Trying next group")
except Exception:
sickrage.app.log.debug("Failed while loading group from AniDB. Trying next group")
else:
for line in group.datalines:
if line["shortname"]:
shortGroupList.append(line["shortname"])
else:
if groupName not in shortGroupList:
shortGroupList.append(groupName)
else:
shortGroupList = groups
return shortGroupList
......@@ -113,3 +113,10 @@ class NoFreeSpaceException(SickRageException):
"""
No free space left
"""
class AnidbAdbaConnectionException(SickRageException):
"""
Connection exceptions raised while trying to communicate with the Anidb UDP api.
More info on the api: https://wiki.anidb.net/w/API.
"""
import adba
import sickrage
from adba import AniDBCommandTimeoutError
from sickrage.core.exceptions import AnidbAdbaConnectionException
def set_up_anidb_connection():
"""Connect to anidb."""
if not sickrage.app.config.use_anidb:
sickrage.app.log.debug('Usage of AniDB disabled. Skipping')
return False
if not sickrage.app.config.anidb_username and not sickrage.app.config.anidb_password:
sickrage.app.log.debug('AniDB username and/or password are not set. Aborting anidb lookup.')
return False
if not sickrage.app.adba_connection:
try:
sickrage.app.adba_connection = adba.Connection(keepAlive=True)
except Exception as error:
sickrage.applog.warning('AniDB exception msg: {0!r}'.format(error))
return False
try:
if not sickrage.app.adba_connection.authed():
sickrage.app.adba_connection.auth(sickrage.app.config.anidb_username, sickrage.app.config.anidb_password)
else:
return True
except Exception as error:
sickrage.app.log.warning('AniDB exception msg: {0!r}'.format(error))
return False
return sickrage.app.adba_connection.authed()
def get_release_groups_for_anime(series_name):
"""Get release groups for an anidb anime."""
groups = []
if set_up_anidb_connection():
try:
anime = adba.Anime(sickrage.app.adba_connection, name=series_name)
groups = anime.get_groups()
except Exception as error:
sickrage.app.log.warning('Unable to retrieve Fansub Groups from AniDB. Error: {}'.format(error.message))
raise AnidbAdbaConnectionException(error)
return groups
def get_short_group_name(release_group):
short_group_list = []
try:
group = sickrage.app.adba_connection.group(gname=release_group)
except AniDBCommandTimeoutError:
sickrage.app.log.debug('Timeout while loading group from AniDB. Trying next group')
except Exception:
sickrage.app.log.debug('Failed while loading group from AniDB. Trying next group')
else:
for line in group.datalines:
if line['shortname']:
short_group_list.append(line['shortname'])
else:
if release_group not in short_group_list:
short_group_list.append(release_group)
return short_group_list
def short_group_names(groups):
"""
Find AniDB short group names for release groups
:param groups: list of groups to find short group names for
:return: list of shortened group names
"""
groups = groups.split(",")
short_group_list = []
if set_up_anidb_connection():
for group_name in groups:
short_group_list += get_short_group_name(group_name) or [group_name]
else:
short_group_list = groups
return short_group_list
def get_anime_episode(file_path):
"""
Look up anidb properties for an episode
:param file_path: file to check
:return: episode object
"""
ep = None
if set_up_anidb_connection():
ep = adba.aniDBAbstracter.Episode(sickrage.app.adba_connection, filePath=file_path,
paramsF=[
"quality",
"anidb_file_name",
"crc32"
],
paramsA=[
"epno",
"english_name",
"short_name_list",
"other_name",
"synonym_list"
])
return ep
......@@ -277,16 +277,17 @@ anime_regexes = [
'''),
('anime_standard_round',
# [Stratos-Subs]_Infinite_Stratos_-_12_(1280x720_H.264_AAC)_[379759DB]
# [Stratos-Subs]_Infinite_Stratos_-_12_(1280x720_H.264_AAC) [379759DB]
# [ShinBunBu-Subs] Bleach - 02-03 (CX 1280x720 x264 AAC)
r'''
^(\[(?P<release_group>.+?)\][ ._-]*)? # Release Group and separator
(?P<series_name>.+?)[ ._-]+ # Show_Name and separator
(?P<ep_ab_num>((?!(1080|720|480)[pi])|(?![hx].?264))\d{1,3}) # E01
(-(?P<extra_ab_ep_num>((?!(1080|720|480)[pi])|(?![hx].?264))\d{1,3}))? # E02
(v(?P<version>[0-9]))? # version
[ ._-]+\((?P<extra_info>(CX[ ._-]?)?\d{3,4}[xp]?\d{0,4}[\.\w\s-]*)\) # Source_Quality_Etc-
(\[(?P<crc>\w{8})\])? # CRC
.*? # Separator and EOL
^(\[(?P<release_group>.+?)\][ ._-]*)? # Release Group and separator
(?P<series_name>.+?)[ ._-]+ # Show_Name and separator
(?P<ep_ab_num>((?!(1080|720|480)[pi])|(?![hx].?264))\d{1,3}) # E01
(-(?P<extra_ab_ep_num>((?!(1080|720|480)[pi])|(?![hx].?264))\d{1,3}))? # E02
(v(?P<version>[0-9]))? # version
[ ._-]+\((?P<extra_info>(\w+[ ._-]?)?\d{3,4}[xp]?\d{0,4}[\.\w\s-]*)\)[ ._-]+ # Source_Quality_Etc-
(\[(?P<crc>\w{8})\])? # CRC
.*? # Separator and EOL
'''),
('anime_slash',
# [SGKK] Bleach 312v1 [720p/MKV]
......
......@@ -26,7 +26,6 @@ import stat
import subprocess
import sickrage
from adba import aniDBAbstracter
from sickrage.core.common import Quality, ARCHIVED, DOWNLOADED
from sickrage.core.exceptions import EpisodeNotFoundException, EpisodePostProcessingFailedException, \
NoFreeSpaceException
......@@ -34,6 +33,7 @@ from sickrage.core.helpers import findCertainShow, show_names, replaceExtension,
chmod_as_parent, move_file, copy_file, hardlink_file, move_and_symlink_file, remove_non_release_groups, \
remove_extension, \
isFileLocked, verify_freespace, delete_empty_folders, make_dirs, symlink, is_rar_file, glob_escape, touch_file
from sickrage.core.helpers.anidb import get_anime_episode
from sickrage.core.nameparser import InvalidNameException, InvalidShowException, \
NameParser
from sickrage.core.tv.show.history import FailedHistory, History # memory intensive
......@@ -623,36 +623,22 @@ class PostProcessor(object):
self._finalize(parse_result)
return to_return
@staticmethod
def _build_anidb_episode(connection, filePath):
"""
Look up anidb properties for an episode
:param connection: anidb connection handler
:param filePath: file to check
:return: episode object
"""
ep = aniDBAbstracter.Episode(connection, filePath=filePath,
paramsF=["quality", "anidb_file_name", "crc32"],
paramsA=["epno", "english_name", "short_name_list", "other_name", "synonym_list"])
return ep
def _add_to_anidb_mylist(self, filePath):
"""
Adds an episode to anidb mylist
:param filePath: file to add to mylist
"""
if sickrage.app.adba_connection:
if not self.anidbEpisode: # seems like we could parse the name before, now lets build the anidb object
self.anidbEpisode = self._build_anidb_episode(sickrage.app.adba_connection, filePath)
self._log("Adding the file to the anidb mylist", sickrage.app.log.DEBUG)
try:
self.anidbEpisode.add_to_mylist(status=1) # status = 1 sets the status of the file to "internal HDD"
except Exception as e:
self._log("exception msg: " + str(e))
if not self.anidbEpisode: # seems like we could parse the name before, now lets build the anidb object
self.anidbEpisode = get_anime_episode(filePath)
if self.anidbEpisode:
self._log("Adding the file to the anidb mylist", sickrage.app.log.DEBUG)
try:
# status of 1 sets the status of the file to "internal HDD"
self.anidbEpisode.add_to_mylist(status=1)
except Exception as e:
self._log("exception msg: " + str(e))
def _find_info(self):
"""
......
......@@ -20,10 +20,10 @@ class RSSCacheUpdater(object):
for providerID, providerObj in sickrage.app.search_providers.sort().items():
if providerObj.isEnabled:
sickrage.app.log.info("Updating RSS cache for provider: [{}]".format(providerObj.name))
sickrage.app.log.debug("Updating RSS cache for provider: [{}]".format(providerObj.name))
threading.currentThread().setName(self.name + "::[" + providerObj.name + "]")
providerObj.cache.update(force)
threading.currentThread().setName(self.name)
sickrage.app.log.info("Updated RSS cache for provider: [{}]".format(providerObj.name))
sickrage.app.log.debug("Updated RSS cache for provider: [{}]".format(providerObj.name))
self.amActive = False
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -41,24 +41,23 @@ from tornado.web import RequestHandler, authenticated
import sickrage
import sickrage.subtitles
from adba import aniDBAbstracter
from sickrage.clients import getClientIstance
from sickrage.clients.sabnzbd import SabNZBd
from sickrage.core import API, google_drive
from sickrage.core.blackandwhitelist import BlackAndWhiteList, \
short_group_names
from sickrage.core.blackandwhitelist import BlackAndWhiteList
from sickrage.core.classes import ErrorViewer, AllShowsUI
from sickrage.core.classes import WarningViewer
from sickrage.core.common import FAILED, IGNORED, Overview, Quality, SKIPPED, \
SNATCHED, UNAIRED, WANTED, cpu_presets, statusStrings
from sickrage.core.exceptions import CantRefreshShowException, \
CantUpdateShowException, EpisodeDeletedException, \
NoNFOException, CantRemoveShowException
NoNFOException, CantRemoveShowException, AnidbAdbaConnectionException
from sickrage.core.helpers import argToBool, backupSR, chmod_as_parent, findCertainShow, generateApiKey, \
getDiskSpaceUsage, makeDir, readFileBuffered, \
remove_article, restoreConfigZip, \
sanitizeFileName, clean_url, try_int, torrent_webui_url, checkbox_to_value, clean_host, \
clean_hosts, app_statistics
from sickrage.core.helpers.anidb import short_group_names, get_release_groups_for_anime
from sickrage.core.helpers.browser import foldersAtPath
from sickrage.core.helpers.compat import cmp
from sickrage.core.helpers.srdatetime import srDateTime
......@@ -1345,7 +1344,6 @@ class Home(WebHandler):
if anyQualities is None:
anyQualities = []
anidb_failed = False
if show is None:
errString = _("Invalid show ID: ") + str(show)
if directCall:
......@@ -1370,15 +1368,10 @@ class Home(WebHandler):
whitelist = showObj.release_groups.whitelist
blacklist = showObj.release_groups.blacklist
if sickrage.app.adba_connection and not anidb_failed:
try:
anime = aniDBAbstracter.Anime(sickrage.app.adba_connection, name=showObj.name)
groups = anime.get_groups()
except Exception as e:
anidb_failed = True
sickrage.app.alerts.error(_('Unable to retreive Fansub Groups from AniDB.'))
sickrage.app.log.debug(
'Unable to retreive Fansub Groups from AniDB. Error is {}'.format(e))
try:
groups = get_release_groups_for_anime(showObj.name)
except AnidbAdbaConnectionException as e:
sickrage.app.log.debug('Unable to get ReleaseGroups: {}'.format(e))
with showObj.lock:
scene_exceptions = get_scene_exceptions(showObj.indexerid)
......@@ -2230,11 +2223,14 @@ class Home(WebHandler):
@staticmethod
def fetch_releasegroups(show_name):
sickrage.app.log.info('ReleaseGroups: %s' % show_name)
if sickrage.app.adba_connection:
anime = aniDBAbstracter.Anime(sickrage.app.adba_connection, name=show_name)
groups = anime.get_groups()
sickrage.app.log.info('ReleaseGroups: %s' % groups)
sickrage.app.log.info('ReleaseGroups: {}'.format(show_name))
try:
groups = get_release_groups_for_anime(show_name)
sickrage.app.log.info('ReleaseGroups: {}'.format(groups))
except AnidbAdbaConnectionException as e:
sickrage.app.log.debug('Unable to get ReleaseGroups: {}'.format(e))
else:
return json_encode({'result': 'success', 'groups': groups})
return json_encode({'result': 'failure'})
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -36,9 +37,5 @@ class AniDBPacketCorruptedError(AniDBError):
pass
class AniDBBannedError(AniDBError):
pass
class AniDBInternalError(AniDBError):
pass
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -15,30 +16,40 @@
# You should have received a copy of the GNU General Public License
# along with aDBa. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import hashlib
import logging
import os
import pickle
import sys
import time
import xml.etree.cElementTree as etree
try:
import xml.etree.cElementTree as etree
except ImportError:
import xml.etree.ElementTree as etree
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
import requests
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
def get_file_hash(filePath):
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
def get_ED2K(filePath, forceHash=False, cacheLocation=os.path.normpath(sys.path[0] + os.sep + "ED2KCache.pickle")):
""" Returns the ed2k hash of a given file."""
if not filePath:
return None
md4 = hashlib.new('md4').copy
ed2k_chunk_size = 9728000
try:
get_ED2K.ED2KCache
except:
if os.path.isfile(cacheLocation):
with open(cacheLocation, 'rb') as f:
get_ED2K.ED2KCache = pickle.load(f)
else:
get_ED2K.ED2KCache = {}
def gen(f):
while True:
x = f.read(9728000)
x = f.read(ed2k_chunk_size)
if x:
yield x
else:
......@@ -49,24 +60,89 @@ def get_file_hash(filePath):
m.update(data)
return m
with open(filePath, 'rb') as f:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexdigest()
def writeCacheToDisk():
try:
if len(get_ED2K.ED2KCache) != 0:
with open(cacheLocation, 'wb') as f:
pickle.dump(get_ED2K.ED2KCache, f, pickle.HIGHEST_PROTOCOL)
except:
logger.error("Error occurred while writing back to disk")
return
file_modified_time = os.path.getmtime(filePath)
file_name = os.path.basename(filePath)
try:
cached_file_modified_time = get_ED2K.ED2KCache[file_name][1]
except:
# if not existing in cache it will be caught by other test
cached_file_modified_time = file_modified_time
if forceHash or file_modified_time > cached_file_modified_time or file_name not in get_ED2K.ED2KCache:
with open(filePath, 'rb') as f:
file_size = os.path.getsize(filePath)
# if file size is small enough the ed2k hash is the same as the md4 hash
if file_size <= ed2k_chunk_size:
full_file = f.read()
new_hash = md4_hash(full_file).hexdigest()
else:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
combinedhash = bytearray()
for hash in hashes:
combinedhash.extend(hash)
new_hash = md4_hash(combinedhash).hexdigest()
get_ED2K.ED2KCache[file_name] = (new_hash, file_modified_time)
writeCacheToDisk()
return new_hash
else:
return get_ED2K.ED2KCache[file_name][0]
def get_file_size(path):
size = os.path.getsize(path)
return size
def read_anidb_xml(file_path=None):
if not file_path:
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
elif not file_path.endswith("xml"):
file_path = os.path.join(file_path, "animetitles.xml")
return read_xml_into_etree(file_path)
def read_tvdb_map_xml(file_path=None):
if not file_path:
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml")
elif not file_path.endswith(".xml"):
file_path = os.path.join(file_path, "anime-list.xml")
return read_xml_into_etree(file_path)
def read_xml_into_etree(filePath):
if not filePath:
return None
if not os.path.isfile(filePath):
if not get_anime_titles_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_titles_xml(filePath):
return
f = open(filePath, "r")
xml_a_setree = etree.ElementTree(file=f)
return xml_a_setree
def _remove_file_failed(file):
try:
os.remove(file)
except:
pass
except OSError:
logger.warning("Error occurred while trying to remove file %s", file)
def download_file(url, filename):
try:
......@@ -86,47 +162,10 @@ def download_file(url, filename):
return True
def get_anime_titles_xml(path):
return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/animetitles.xml", path)
def get_anime_list_xml(path):
return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml", path)
def read_anidb_xml(filePath=None):
if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
if not os.path.isfile(filePath):
if not get_anime_titles_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_titles_xml(filePath):
return
return read_xml_into_etree(filePath)
def read_tvdb_map_xml(filePath=None):
if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml")
if not os.path.isfile(filePath):
if not get_anime_list_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_list_xml(filePath):
return
return read_xml_into_etree(filePath)
def read_xml_into_etree(filePath):
if not filePath:
return None
with open(filePath, "r") as f:
return etree.ElementTree(file=f)
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -15,18 +16,24 @@
# You should have received a copy of the GNU General Public License
# along with aDBa. If not, see <http://www.gnu.org/licenses/>.
import logging
import socket
import sys
import threading
import zlib
from time import time, sleep
from aniDBerrors import *
from aniDBresponses import ResponseResolver
from builtins import bytes
from .aniDBerrors import *
from .aniDBresponses import ResponseResolver
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
class AniDBLink(threading.Thread):
def __init__(self, server, port, myport, logFunction, delay=2, timeout=20, logPrivate=False):
def __init__(self, server, port, myport, delay=2, timeout=20):
super(AniDBLink, self).__init__()
self.server = server
self.port = port
......@@ -36,7 +43,7 @@ class AniDBLink(threading.Thread):
self.myport = 0
self.bound = self.connectSocket(myport, self.timeout)
self.cmd_queue = {None:None}
self.cmd_queue = {None: None}
self.resp_tagged_queue = {}
self.resp_untagged_queue = []
self.tags = []
......@@ -46,11 +53,11 @@ class AniDBLink(threading.Thread):
self.banned = False
self.crypt = None
self.log = logFunction
self.logPrivate = logPrivate
self._stop = threading.Event()
self._quiting = False
self.QuitProcessed = False
self.setDaemon(True)
self.start()
......@@ -70,20 +77,19 @@ class AniDBLink(threading.Thread):
return False
def disconnectSocket(self):
self.sock.close()
self.sock.shutdown(socket.SHUT_RD)
# close is not called as the garbage collection from python will handle this for us. Calling close can also cause issues with the threaded code.
# self.sock.close()
def stop (self):