Commit c26e5b16 authored by echel0n's avatar echel0n

Refactored AniDB code.

parent 41039e5c
# Changelog
- * 3a00fef - 2019-02-24: Fixed KeyError for twitter notifier.
- * ca574bf - 2019-02-24: Refactored AniDB code.
- * 41039e5 - 2019-02-24: Fixed KeyError for twitter notifier.
- * 135d868 - 2019-02-24: Fixed AttributeError for gktorrent provider. Refactored log messages for rss cache updater to debug.
- * ffd8fa4 - 2019-02-24: Release v9.4.73
- * 47572b6 - 2019-02-24: Release v9.4.72
......
......@@ -39,7 +39,6 @@ from fake_useragent import UserAgent
from keycloak.realm import KeycloakRealm
from tornado.ioloop import IOLoop
import adba
import sickrage
from sickrage.core.api import API
from sickrage.core.caches.name_cache import NameCache
......@@ -296,17 +295,6 @@ class Core(object):
except Exception:
continue
# init anidb connection
if self.config.use_anidb:
def anidb_logger(msg):
return self.log.debug("AniDB: {} ".format(msg))
try:
self.adba_connection = adba.Connection(keepAlive=True, log=anidb_logger)
self.adba_connection.auth(self.config.anidb_username, self.config.anidb_password)
except Exception as e:
self.log.warning("AniDB exception msg: %r " % repr(e))
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
......
......@@ -20,7 +20,6 @@
from __future__ import unicode_literals
import sickrage
from adba.aniDBerrors import AniDBCommandTimeoutError
class BlackAndWhiteList(object):
......@@ -87,7 +86,6 @@ class BlackAndWhiteList(object):
if dbData:
sickrage.app.main_db.delete(sickrage.app.main_db.get(table, self.show_id))
def _load_list(self, table):
"""
DB: Fetch keywords for current show
......@@ -143,32 +141,3 @@ class BlackAndWhiteList(object):
class BlackWhitelistNoShowIDException(Exception):
"""No show_id was given"""
def short_group_names(groups):
"""
Find AniDB short group names for release groups
:param groups: list of groups to find short group names for
:return: list of shortened group names
"""
groups = groups.split(",")
shortGroupList = []
if sickrage.app.adba_connection:
for groupName in groups:
try:
group = sickrage.app.adba_connection.group(gname=groupName)
except AniDBCommandTimeoutError:
sickrage.app.log.debug("Timeout while loading group from AniDB. Trying next group")
except Exception:
sickrage.app.log.debug("Failed while loading group from AniDB. Trying next group")
else:
for line in group.datalines:
if line["shortname"]:
shortGroupList.append(line["shortname"])
else:
if groupName not in shortGroupList:
shortGroupList.append(groupName)
else:
shortGroupList = groups
return shortGroupList
......@@ -113,3 +113,10 @@ class NoFreeSpaceException(SickRageException):
"""
No free space left
"""
class AnidbAdbaConnectionException(SickRageException):
"""
Connection exceptions raised while trying to communicate with the Anidb UDP api.
More info on the api: https://wiki.anidb.net/w/API.
"""
import adba
import sickrage
from adba import AniDBCommandTimeoutError
from sickrage.core.exceptions import AnidbAdbaConnectionException
def set_up_anidb_connection():
"""Connect to anidb."""
if not sickrage.app.config.use_anidb:
sickrage.app.log.debug('Usage of AniDB disabled. Skipping')
return False
if not sickrage.app.config.anidb_username and not sickrage.app.config.anidb_password:
sickrage.app.log.debug('AniDB username and/or password are not set. Aborting anidb lookup.')
return False
if not sickrage.app.adba_connection:
try:
sickrage.app.adba_connection = adba.Connection(keepAlive=True)
except Exception as error:
sickrage.applog.warning('AniDB exception msg: {0!r}'.format(error))
return False
try:
if not sickrage.app.adba_connection.authed():
sickrage.app.adba_connection.auth(sickrage.app.config.anidb_username, sickrage.app.config.anidb_password)
else:
return True
except Exception as error:
sickrage.app.log.warning('AniDB exception msg: {0!r}'.format(error))
return False
return sickrage.app.adba_connection.authed()
def get_release_groups_for_anime(series_name):
"""Get release groups for an anidb anime."""
groups = []
if set_up_anidb_connection():
try:
anime = adba.Anime(sickrage.app.adba_connection, name=series_name)
groups = anime.get_groups()
except Exception as error:
sickrage.app.log.warning('Unable to retrieve Fansub Groups from AniDB. Error: {}'.format(error.message))
raise AnidbAdbaConnectionException(error)
return groups
def get_short_group_name(release_group):
short_group_list = []
try:
group = sickrage.app.adba_connection.group(gname=release_group)
except AniDBCommandTimeoutError:
sickrage.app.log.debug('Timeout while loading group from AniDB. Trying next group')
except Exception:
sickrage.app.log.debug('Failed while loading group from AniDB. Trying next group')
else:
for line in group.datalines:
if line['shortname']:
short_group_list.append(line['shortname'])
else:
if release_group not in short_group_list:
short_group_list.append(release_group)
return short_group_list
def short_group_names(groups):
"""
Find AniDB short group names for release groups
:param groups: list of groups to find short group names for
:return: list of shortened group names
"""
groups = groups.split(",")
short_group_list = []
if set_up_anidb_connection():
for group_name in groups:
short_group_list += get_short_group_name(group_name) or [group_name]
else:
short_group_list = groups
return short_group_list
......@@ -34,6 +34,7 @@ from sickrage.core.helpers import findCertainShow, show_names, replaceExtension,
chmod_as_parent, move_file, copy_file, hardlink_file, move_and_symlink_file, remove_non_release_groups, \
remove_extension, \
isFileLocked, verify_freespace, delete_empty_folders, make_dirs, symlink, is_rar_file, glob_escape, touch_file
from sickrage.core.helpers.anidb import set_up_anidb_connection
from sickrage.core.nameparser import InvalidNameException, InvalidShowException, \
NameParser
from sickrage.core.tv.show.history import FailedHistory, History # memory intensive
......@@ -644,7 +645,7 @@ class PostProcessor(object):
:param filePath: file to add to mylist
"""
if sickrage.app.adba_connection:
if set_up_anidb_connection():
if not self.anidbEpisode: # seems like we could parse the name before, now lets build the anidb object
self.anidbEpisode = self._build_anidb_episode(sickrage.app.adba_connection, filePath)
......
......@@ -41,24 +41,23 @@ from tornado.web import RequestHandler, authenticated
import sickrage
import sickrage.subtitles
from adba import aniDBAbstracter
from sickrage.clients import getClientIstance
from sickrage.clients.sabnzbd import SabNZBd
from sickrage.core import API, google_drive
from sickrage.core.blackandwhitelist import BlackAndWhiteList, \
short_group_names
from sickrage.core.blackandwhitelist import BlackAndWhiteList
from sickrage.core.classes import ErrorViewer, AllShowsUI
from sickrage.core.classes import WarningViewer
from sickrage.core.common import FAILED, IGNORED, Overview, Quality, SKIPPED, \
SNATCHED, UNAIRED, WANTED, cpu_presets, statusStrings
from sickrage.core.exceptions import CantRefreshShowException, \
CantUpdateShowException, EpisodeDeletedException, \
NoNFOException, CantRemoveShowException
NoNFOException, CantRemoveShowException, AnidbAdbaConnectionException
from sickrage.core.helpers import argToBool, backupSR, chmod_as_parent, findCertainShow, generateApiKey, \
getDiskSpaceUsage, makeDir, readFileBuffered, \
remove_article, restoreConfigZip, \
sanitizeFileName, clean_url, try_int, torrent_webui_url, checkbox_to_value, clean_host, \
clean_hosts, app_statistics
from sickrage.core.helpers.anidb import short_group_names, get_release_groups_for_anime
from sickrage.core.helpers.browser import foldersAtPath
from sickrage.core.helpers.compat import cmp
from sickrage.core.helpers.srdatetime import srDateTime
......@@ -1345,7 +1344,6 @@ class Home(WebHandler):
if anyQualities is None:
anyQualities = []
anidb_failed = False
if show is None:
errString = _("Invalid show ID: ") + str(show)
if directCall:
......@@ -1370,15 +1368,10 @@ class Home(WebHandler):
whitelist = showObj.release_groups.whitelist
blacklist = showObj.release_groups.blacklist
if sickrage.app.adba_connection and not anidb_failed:
try:
anime = aniDBAbstracter.Anime(sickrage.app.adba_connection, name=showObj.name)
groups = anime.get_groups()
except Exception as e:
anidb_failed = True
sickrage.app.alerts.error(_('Unable to retreive Fansub Groups from AniDB.'))
sickrage.app.log.debug(
'Unable to retreive Fansub Groups from AniDB. Error is {}'.format(e))
try:
groups = get_release_groups_for_anime(showObj.name)
except AnidbAdbaConnectionException as e:
sickrage.app.log.debug('Unable to get ReleaseGroups: {}'.format(e))
with showObj.lock:
scene_exceptions = get_scene_exceptions(showObj.indexerid)
......@@ -2230,11 +2223,14 @@ class Home(WebHandler):
@staticmethod
def fetch_releasegroups(show_name):
sickrage.app.log.info('ReleaseGroups: %s' % show_name)
if sickrage.app.adba_connection:
anime = aniDBAbstracter.Anime(sickrage.app.adba_connection, name=show_name)
groups = anime.get_groups()
sickrage.app.log.info('ReleaseGroups: %s' % groups)
sickrage.app.log.info('ReleaseGroups: {}'.format(show_name))
try:
groups = get_release_groups_for_anime(show_name)
sickrage.app.log.info('ReleaseGroups: {}'.format(groups))
except AnidbAdbaConnectionException as e:
sickrage.app.log.debug('Unable to get ReleaseGroups: {}'.format(e))
else:
return json_encode({'result': 'success', 'groups': groups})
return json_encode({'result': 'failure'})
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -36,9 +37,5 @@ class AniDBPacketCorruptedError(AniDBError):
pass
class AniDBBannedError(AniDBError):
pass
class AniDBInternalError(AniDBError):
pass
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -15,30 +16,40 @@
# You should have received a copy of the GNU General Public License
# along with aDBa. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import hashlib
import logging
import os
import pickle
import sys
import time
import xml.etree.cElementTree as etree
try:
import xml.etree.cElementTree as etree
except ImportError:
import xml.etree.ElementTree as etree
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
import requests
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
def get_file_hash(filePath):
# http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/
def get_ED2K(filePath, forceHash=False, cacheLocation=os.path.normpath(sys.path[0] + os.sep + "ED2KCache.pickle")):
""" Returns the ed2k hash of a given file."""
if not filePath:
return None
md4 = hashlib.new('md4').copy
ed2k_chunk_size = 9728000
try:
get_ED2K.ED2KCache
except:
if os.path.isfile(cacheLocation):
with open(cacheLocation, 'rb') as f:
get_ED2K.ED2KCache = pickle.load(f)
else:
get_ED2K.ED2KCache = {}
def gen(f):
while True:
x = f.read(9728000)
x = f.read(ed2k_chunk_size)
if x:
yield x
else:
......@@ -49,24 +60,89 @@ def get_file_hash(filePath):
m.update(data)
return m
with open(filePath, 'rb') as f:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
else:
return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexdigest()
def writeCacheToDisk():
try:
if len(get_ED2K.ED2KCache) != 0:
with open(cacheLocation, 'wb') as f:
pickle.dump(get_ED2K.ED2KCache, f, pickle.HIGHEST_PROTOCOL)
except:
logger.error("Error occurred while writing back to disk")
return
file_modified_time = os.path.getmtime(filePath)
file_name = os.path.basename(filePath)
try:
cached_file_modified_time = get_ED2K.ED2KCache[file_name][1]
except:
# if not existing in cache it will be caught by other test
cached_file_modified_time = file_modified_time
if forceHash or file_modified_time > cached_file_modified_time or file_name not in get_ED2K.ED2KCache:
with open(filePath, 'rb') as f:
file_size = os.path.getsize(filePath)
# if file size is small enough the ed2k hash is the same as the md4 hash
if file_size <= ed2k_chunk_size:
full_file = f.read()
new_hash = md4_hash(full_file).hexdigest()
else:
a = gen(f)
hashes = [md4_hash(data).digest() for data in a]
combinedhash = bytearray()
for hash in hashes:
combinedhash.extend(hash)
new_hash = md4_hash(combinedhash).hexdigest()
get_ED2K.ED2KCache[file_name] = (new_hash, file_modified_time)
writeCacheToDisk()
return new_hash
else:
return get_ED2K.ED2KCache[file_name][0]
def get_file_size(path):
size = os.path.getsize(path)
return size
def read_anidb_xml(file_path=None):
if not file_path:
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
elif not file_path.endswith("xml"):
file_path = os.path.join(file_path, "animetitles.xml")
return read_xml_into_etree(file_path)
def read_tvdb_map_xml(file_path=None):
if not file_path:
file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml")
elif not file_path.endswith(".xml"):
file_path = os.path.join(file_path, "anime-list.xml")
return read_xml_into_etree(file_path)
def read_xml_into_etree(filePath):
if not filePath:
return None
if not os.path.isfile(filePath):
if not get_anime_titles_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_titles_xml(filePath):
return
f = open(filePath, "r")
xml_a_setree = etree.ElementTree(file=f)
return xml_a_setree
def _remove_file_failed(file):
try:
os.remove(file)
except:
pass
except OSError:
logger.warning("Error occurred while trying to remove file %s", file)
def download_file(url, filename):
try:
......@@ -86,47 +162,10 @@ def download_file(url, filename):
return True
def get_anime_titles_xml(path):
return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/animetitles.xml", path)
def get_anime_list_xml(path):
return download_file("https://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml", path)
def read_anidb_xml(filePath=None):
if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "animetitles.xml")
if not os.path.isfile(filePath):
if not get_anime_titles_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_titles_xml(filePath):
return
return read_xml_into_etree(filePath)
def read_tvdb_map_xml(filePath=None):
if not filePath:
filePath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anime-list.xml")
if not os.path.isfile(filePath):
if not get_anime_list_xml(filePath):
return
else:
mtime = os.path.getmtime(filePath)
if time.time() > mtime + 24 * 60 * 60:
if not get_anime_list_xml(filePath):
return
return read_xml_into_etree(filePath)
def read_xml_into_etree(filePath):
if not filePath:
return None
with open(filePath, "r") as f:
return etree.ElementTree(file=f)
#!/usr/bin/env python
# coding=utf-8
#
# This file is part of aDBa.
#
......@@ -15,18 +16,24 @@
# You should have received a copy of the GNU General Public License
# along with aDBa. If not, see <http://www.gnu.org/licenses/>.
import logging
import socket
import sys
import threading
import zlib
from time import time, sleep
from aniDBerrors import *
from aniDBresponses import ResponseResolver
from builtins import bytes
from .aniDBerrors import *
from .aniDBresponses import ResponseResolver
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
class AniDBLink(threading.Thread):
def __init__(self, server, port, myport, logFunction, delay=2, timeout=20, logPrivate=False):
def __init__(self, server, port, myport, delay=2, timeout=20):
super(AniDBLink, self).__init__()
self.server = server
self.port = port
......@@ -36,7 +43,7 @@ class AniDBLink(threading.Thread):
self.myport = 0
self.bound = self.connectSocket(myport, self.timeout)
self.cmd_queue = {None:None}
self.cmd_queue = {None: None}
self.resp_tagged_queue = {}
self.resp_untagged_queue = []
self.tags = []
......@@ -46,11 +53,11 @@ class AniDBLink(threading.Thread):
self.banned = False
self.crypt = None
self.log = logFunction
self.logPrivate = logPrivate
self._stop = threading.Event()
self._quiting = False
self.QuitProcessed = False
self.setDaemon(True)
self.start()
......@@ -70,20 +77,19 @@ class AniDBLink(threading.Thread):
return False
def disconnectSocket(self):
self.sock.close()
self.sock.shutdown(socket.SHUT_RD)
# close is not called as the garbage collection from python will handle this for us. Calling close can also cause issues with the threaded code.
# self.sock.close()
def stop (self):
self.log("Releasing socket and stopping link thread")
def stop(self):
logger.info("Releasing socket and stopping link thread")
self._quiting = True
self.disconnectSocket()
self._stop.set()
def stopped (self):
def stopped(self):
return self._stop.isSet()
def print_log(self, data):
print data
def print_log_dummy(self, data):
pass
......@@ -93,40 +99,44 @@ class AniDBLink(threading.Thread):
data = self.sock.recv(8192)
except socket.timeout:
self._handle_timeouts()
continue
self.log("NetIO < %s" % repr(data))
except OSError as error:
logger.exception('Exception: %s', error)
break
logger.debug("NetIO < %r", data)
try:
for i in range(2):
try:
tmp = data
resp = None
if tmp[:2] == '\x00\x00':
if tmp[:2] == b'\x00\x00':
tmp = zlib.decompressobj().decompress(tmp[2:])
self.log("UnZip | %s" % repr(tmp))
logger.debug("UnZip | %r", tmp)
resp = ResponseResolver(tmp)
except:
except Exception as e:
logger.exception('Exception: %s', e)
sys.excepthook(*sys.exc_info())
self.crypt = None
self.session = None
else:
break
if not resp:
raise AniDBPacketCorruptedError, "Either decrypting, decompressing or parsing the packet failed"
raise AniDBPacketCorruptedError("Either decrypting, decompressing or parsing the packet failed")
cmd = self._cmd_dequeue(resp)
resp = resp.resolve(cmd)
resp.parse()
if resp.rescode in ('200', '201'):
self.session = resp.attrs['sesskey']
if resp.rescode in ('209',):
raise Exception, "sorry encryption is not supported"
#self.crypt=aes(md5(resp.req.apipassword+resp.attrs['salt']).digest())
if resp.rescode in ('203', '403', '500', '501', '503', '506'):
if resp.rescode in (b'200', b'201'):
self.session = resp.attrs[b'sesskey']
if resp.rescode in (b'209',):
logger.error("sorry encryption is not supported")
raise AniDBError()
# self.crypt=aes(md5(resp.req.apipassword+resp.attrs['salt']).digest())
if resp.rescode in (b'203', b'403', b'500', b'501', b'503', b'506'):
self.session = None
self.crypt = None
if resp.rescode in ('504', '555'):
if resp.rescode in (b'504', b'555'):
self.banned = True
print "AniDB API informs that user or client is banned:", resp.resstr
logger.critical(("AniDB API informs that user or client is banned:", resp.resstr))
resp.handle()
if not cmd or not cmd.mode:
self._resp_queue(resp)
......@@ -134,18 +144,24 @@ class AniDBLink(threading.Thread):
self.tags.remove(resp.restag)
except:
sys.excepthook(*sys.exc_info())
print "Avoiding flood by paranoidly panicing: Aborting link thread, killing connection, releasing waiters and quiting"
logger.error("Avoiding flood by paranoidly panicing: Aborting link thread, killing connection, releasing waiters and quiting")
self.sock.close()
try:cmd.waiter.release()
except:pass
for tag, cmd in self.cmd_queue.iteritems():
try:cmd.waiter.release()
except:pass
try:
cmd.waiter.release()
except:
pass
for tag, cmd in self.cmd_queue.items():
try:
cmd.waiter.release()
except:
pass
sys.exit()
if self._quiting:
self.QuitProcessed = True
def _handle_timeouts(self):
willpop = []
for tag, cmd in self.cmd_queue.iteritems():
for tag, cmd in self.cmd_queue.items():
if not tag:
continue
if time() - cmd.started > self.timeout:
......@@ -181,7 +197,7 @@ class AniDBLink(threading.Thread):
return self.cmd_queue.pop(resp.restag)
def _delay(self):
return (self.delay < 2.1 and 2.1 or self.delay)
return self.delay < 2.1 and 2.1 or self.delay
def _do_delay(self):
age = time() - self.lastpacket
......@@ -191,30 +207,28 @@ class AniDBLink(threading.Thread):
def _send(self, command):
if self.banned:
self.log("NetIO | BANNED")
raise AniDBBannedError, "Not sending, banned"
logger.debug("NetIO | BANNED")
raise AniDBError("Not sending, banned")
self._do_delay()
self.lastpacket = time()
command.started = time()
data = command.raw_data()
self.sock.sendto(data, self.target)
if command.command == 'AUTH' and self.logPrivate:
self.log("NetIO > sensitive data is not logged!")
else:
self.log("NetIO > %s" % repr(data))
self.sock.sendto(bytes(data, b"ASCII"), self.target)
if command.command == b'AUTH':
logger.debug("NetIO > sensitive data is not logged!")
def new_tag(self):
if not len(self.tags):
maxtag = "T000"
maxtag = b"T000"
else:
maxtag = max(self.tags)
newtag = "T%03d" % (int(maxtag[1:]) + 1)
newtag = b"T%03d" % (int(maxtag[1:]) + 1)
return newtag
def request(self, command):
if not (self.session and command.session) and command.command not in ('AUTH', 'PING', 'ENCRYPT'):
raise AniDBMustAuthError, "You must be authed to execute commands besides AUTH and PING"
if not (self.session and command.session) and command.command not in (b'AUTH', b'PING', b'ENCRYPT'):