__init__.py 50.5 KB
Newer Older
1
# Author: echel0n <[email protected]>
echel0n's avatar
echel0n committed
2
# URL: https://sickrage.ca
3
#
echel0n's avatar
echel0n committed
4
# This file is part of SickRage.
5
#
echel0n's avatar
echel0n committed
6
# SickRage is free software: you can redistribute it and/or modify
7 8 9 10
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
echel0n's avatar
echel0n committed
11
# SickRage is distributed in the hope that it will be useful,
12 13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
15 16
#
# You should have received a copy of the GNU General Public License
echel0n's avatar
echel0n committed
17
# along with SickRage.  If not, see <http://www.gnu.org/licenses/>.
18

19 20
from __future__ import unicode_literals

21
import datetime
22 23
import importlib
import io
24 25 26 27
import itertools
import os
import random
import re
echel0n's avatar
echel0n committed
28
from base64 import b16encode, b32decode, b64decode
echel0n's avatar
echel0n committed
29
from collections import OrderedDict, defaultdict
30 31
from time import sleep
from urlparse import urljoin
32
from xml.sax import SAXParseException
33

34 35
import bencode
from feedparser import FeedParserDict
echel0n's avatar
echel0n committed
36
from requests.utils import add_dict_to_cookiejar, dict_from_cookiejar
37 38

import sickrage
echel0n's avatar
echel0n committed
39
from sickrage.core.api.cache import TorrentCacheAPI
40
from sickrage.core.caches.tv_cache import TVCache
41
from sickrage.core.classes import NZBSearchResult, SearchResult, TorrentSearchResult
42
from sickrage.core.common import MULTI_EP_RESULT, Quality, SEASON_RESULT, cpu_presets
43
from sickrage.core.helpers import chmod_as_parent, findCertainShow, sanitizeFileName, clean_url, bs4_parser, \
44
    validate_url, try_int, convert_size
45
from sickrage.core.helpers.show_names import allPossibleShowNames
46
from sickrage.core.nameparser import InvalidNameException, InvalidShowException, NameParser
47
from sickrage.core.scene_exceptions import get_scene_exceptions
echel0n's avatar
echel0n committed
48
from sickrage.core.websession import WebSession
49

50

51
class GenericProvider(object):
52
    def __init__(self, name, url, private):
53
        self.name = name
54
        self.urls = {'base_url': url}
55
        self.private = private
56
        self.supports_backlog = True
echel0n's avatar
echel0n committed
57
        self.supports_absolute_numbering = False
58
        self.anime_only = False
echel0n's avatar
echel0n committed
59
        self.search_mode = 'eponly'
60 61 62 63
        self.search_fallback = False
        self.enabled = False
        self.enable_daily = False
        self.enable_backlog = False
64
        self.cache = TVCache(self)
65
        self.proper_strings = ['PROPER|REPACK|REAL|RERIP']
echel0n's avatar
echel0n committed
66
        self.search_separator = ' '
67

68
        # cookies
69 70 71
        self.enable_cookies = False
        self.cookies = ''

echel0n's avatar
echel0n committed
72 73
        # web session
        self.session = WebSession(cloudflare=True)
echel0n's avatar
echel0n committed
74

75 76
    @property
    def id(self):
77
        return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower()))
78

79 80 81
    @property
    def isEnabled(self):
        return self.enabled
82

83
    @property
84
    def imageName(self):
85
        return ""
86

echel0n's avatar
echel0n committed
87 88 89 90
    @property
    def seed_ratio(self):
        return ''

91 92
    @property
    def isAlive(self):
93
        return True
94

95
    def _check_auth(self):
96 97
        return True

98
    def login(self):
99
        return True
100

101 102 103 104 105 106 107
    @classmethod
    def get_subclasses(cls):
        yield cls
        if cls.__subclasses__():
            for sub in cls.__subclasses__():
                for s in sub.get_subclasses():
                    yield s
108

109
    def getResult(self, episodes=None):
110 111 112
        """
        Returns a result of the correct type for this provider
        """
113
        return SearchResult(episodes)
114

echel0n's avatar
echel0n committed
115
    def get_content(self, url):
echel0n's avatar
echel0n committed
116
        if self.login():
echel0n's avatar
echel0n committed
117 118 119
            headers = {}
            if url.startswith('http'):
                headers = {'Referer': '/'.join(url.split('/')[:3]) + '/'}
echel0n's avatar
echel0n committed
120

121 122 123 124 125
            if not url.startswith('magnet'):
                try:
                    return self.session.get(url, verify=False, headers=headers).content
                except Exception:
                    pass
126

127 128
    def make_filename(self, name):
        return ""
129

130
    def get_quality(self, item, anime=False):
131 132 133 134 135 136 137 138 139 140 141
        """
        Figures out the quality of the given RSS item node

        item: An elementtree.ElementTree element representing the <item> tag of the RSS feed

        Returns a Quality value obtained from the node's data
        """
        (title, url) = self._get_title_and_url(item)
        quality = Quality.sceneQuality(title, anime)
        return quality

echel0n's avatar
echel0n committed
142
    def search(self, search_strings, age=0, ep_obj=None, **kwargs):
143 144 145
        return []

    def _get_season_search_strings(self, episode):
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
        """
        Get season search strings.
        """
        search_string = {
            'Season': []
        }

        for show_name in allPossibleShowNames(episode.show, episode.scene_season):
            episode_string = show_name + ' '

            if episode.show.air_by_date or episode.show.sports:
                episode_string += str(episode.airdate).split('-')[0]
            elif episode.show.anime:
                episode_string += 'Season'
            else:
                episode_string += 'S{season:0>2}'.format(season=episode.scene_season)

            search_string['Season'].append(episode_string.strip())

        return [search_string]

echel0n's avatar
echel0n committed
167
    def _get_episode_search_strings(self, episode, add_string=''):
168 169 170 171 172 173 174 175 176 177 178
        """
        Get episode search strings.
        """
        if not episode:
            return []

        search_string = {
            'Episode': []
        }

        for show_name in allPossibleShowNames(episode.show, episode.scene_season):
echel0n's avatar
echel0n committed
179
            episode_string = show_name + self.search_separator
180
            episode_string_fallback = None
181 182 183 184 185 186 187 188

            if episode.show.air_by_date:
                episode_string += str(episode.airdate).replace('-', ' ')
            elif episode.show.sports:
                episode_string += str(episode.airdate).replace('-', ' ')
                episode_string += ('|', ' ')[len(self.proper_strings) > 1]
                episode_string += episode.airdate.strftime('%b')
            elif episode.show.anime:
189
                # If the show name is a season scene exception, we want to use the indexer episode number.
190
                if (episode.scene_season > 1 and
191
                        show_name in get_scene_exceptions(episode.show.indexerid, episode.scene_season)):
192 193 194 195 196
                    # This is apparently a season exception, let's use the scene_episode instead of absolute
                    ep = episode.scene_episode
                else:
                    ep = episode.scene_absolute_number
                episode_string += '{episode:0>2}'.format(episode=ep)
197
                episode_string_fallback = episode_string + '{episode:0>3}'.format(episode=ep)
198
            else:
echel0n's avatar
echel0n committed
199
                episode_string += sickrage.app.naming_ep_type[2] % {
200 201 202
                    'seasonnumber': episode.scene_season,
                    'episodenumber': episode.scene_episode,
                }
203

204
            if add_string:
echel0n's avatar
echel0n committed
205
                episode_string += self.search_separator + add_string
206
                episode_string_fallback += self.search_separator + add_string
207 208

            search_string['Episode'].append(episode_string.strip())
209
            if episode_string_fallback:
210 211 212
                search_string['Episode'].append(episode_string_fallback.strip())

        return [search_string]
213 214 215 216 217 218 219 220 221 222

    def _get_title_and_url(self, item):
        """
        Retrieves the title and URL data from the item XML node

        item: An elementtree.ElementTree element representing the <item> tag of the RSS feed

        Returns: A tuple containing two strings representing title and URL respectively
        """

223 224
        title = item.get('title', '').replace(' ', '.')
        url = item.get('link', '').replace('&amp;', '&').replace('%26tr%3D', '&tr=')
225 226 227

        return title, url

228
    def _get_size(self, item):
229
        """Gets the size from the item"""
230
        sickrage.app.log.debug("Provider type doesn't have ability to provide download size implemented yet")
231 232
        return -1

233 234 235 236 237
    def _get_result_stats(self, item):
        # Get seeders/leechers stats
        seeders = item.get('seeders', -1)
        leechers = item.get('leechers', -1)
        return try_int(seeders, -1), try_int(leechers, -1)
238

239
    def findSearchResults(self, show, episodes, search_mode, manualSearch=False, downCurQuality=False, cacheOnly=False):
240 241 242
        results = {}
        itemList = []

echel0n's avatar
echel0n committed
243 244 245
        if not self._check_auth:
            return results

246 247 248
        searched_scene_season = None
        for epObj in episodes:
            # search cache for episode result
echel0n's avatar
echel0n committed
249
            cacheResult = self.cache.search_cache(epObj, manualSearch, downCurQuality)
250 251
            if cacheResult:
                if epObj.episode not in results:
252
                    results[epObj.episode] = cacheResult[epObj.episode]
253
                else:
254
                    results[epObj.episode].extend(cacheResult[epObj.episode])
255 256 257 258 259 260 261 262 263 264 265

                # found result, search next episode
                continue

            # skip if season already searched
            if len(episodes) > 1 and search_mode == 'sponly' and searched_scene_season == epObj.scene_season:
                continue

            # mark season searched for season pack searches so we can skip later on
            searched_scene_season = epObj.scene_season

echel0n's avatar
echel0n committed
266 267 268 269
            # check if this is a cache only search
            if cacheOnly:
                continue

270 271 272 273 274 275 276 277 278 279
            search_strings = []
            if len(episodes) > 1 and search_mode == 'sponly':
                # get season search results
                search_strings = self._get_season_search_strings(epObj)
            elif search_mode == 'eponly':
                # get single episode search results
                search_strings = self._get_episode_search_strings(epObj)

            first = search_strings and isinstance(search_strings[0], dict) and 'rid' in search_strings[0]
            if first:
280
                sickrage.app.log.debug('First search_string has rid')
281 282

            for curString in search_strings:
283
                try:
284
                    itemList += self.search(curString, ep_obj=epObj)
285 286 287
                except SAXParseException:
                    continue

288 289 290
                if first:
                    first = False
                    if itemList:
291
                        sickrage.app.log.debug(
292
                            'First search_string had rid, and returned results, skipping query by string')
293 294
                        break
                    else:
295
                        sickrage.app.log.debug(
296
                            'First search_string had rid, but returned no results, searching with string query')
297 298 299 300 301 302

        # if we found what we needed already from cache then return results and exit
        if len(results) == len(episodes):
            return results

        # sort list by quality
echel0n's avatar
echel0n committed
303 304 305
        if itemList:
            # categorize the items into lists by quality
            items = defaultdict(list)
306
            for item in itemList:
307
                items[self.get_quality(item, anime=show.is_anime)].append(item)
echel0n's avatar
echel0n committed
308 309 310 311 312 313 314 315 316

            # temporarily remove the list of items with unknown quality
            unknown_items = items.pop(Quality.UNKNOWN, [])

            # make a generator to sort the remaining items by descending quality
            items_list = (items[quality] for quality in sorted(items, reverse=True))

            # unpack all of the quality lists into a single sorted list
            items_list = list(itertools.chain(*items_list))
317

echel0n's avatar
echel0n committed
318 319
            # extend the list with the unknown qualities, now sorted at the bottom of the list
            items_list.extend(unknown_items)
320 321 322

        # filter results
        for item in itemList:
323 324
            result = self.getResult()

echel0n's avatar
echel0n committed
325
            result.name, result.url = self._get_title_and_url(item)
326

327 328 329 330
            # ignore invalid urls
            if not validate_url(result.url) and not result.url.startswith('magnet'):
                continue

331
            try:
echel0n's avatar
echel0n committed
332
                parse_result = NameParser(showObj=show, validate_show=True).parse(result.name)
333 334
            except (InvalidNameException, InvalidShowException) as e:
                sickrage.app.log.debug("{}".format(e))
335
                continue
336

337 338 339 340 341 342
            result.show = parse_result.show
            result.quality = parse_result.quality
            result.release_group = parse_result.release_group
            result.version = parse_result.version
            result.size = self._get_size(item)
            result.seeders, result.leechers = self._get_result_stats(item)
343

344
            sickrage.app.log.debug("Adding item from search to cache: {}".format(result.name))
345
            self.cache.addCacheEntry(result.name, result.url, result.seeders, result.leechers, result.size)
346 347 348 349

            if not result.show:
                continue

350
            if not (result.show.air_by_date or result.show.sports):
351 352
                if search_mode == 'sponly':
                    if len(parse_result.episode_numbers):
353 354
                        sickrage.app.log.debug("This is supposed to be a season pack search but the result {} is not "
                                               "a valid season pack, skipping it".format(result.name))
355
                        continue
356
                    if len(parse_result.episode_numbers) and (
357
                            parse_result.season_number not in set([ep.season for ep in episodes])
358
                            or not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]):
359 360
                        sickrage.app.log.debug("The result {} doesn't seem to be a valid episode that we are trying "
                                               "to snatch, ignoring".format(result.name))
361
                        continue
362 363 364 365
                else:
                    if not len(parse_result.episode_numbers) and parse_result.season_number and not [ep for ep in
                                                                                                     episodes if
                                                                                                     ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
366 367
                        sickrage.app.log.debug("The result {} doesn't seem to be a valid season that we are trying to "
                                               "snatch, ignoring".format(result.name))
368
                        continue
369 370
                    elif len(parse_result.episode_numbers) and not [ep for ep in episodes if
                                                                    ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
371 372
                        sickrage.app.log.debug("The result {} doesn't seem to be a valid episode that we are trying "
                                               "to snatch, ignoring".format(result.name))
373
                        continue
374

375 376 377
                # we just use the existing info for normal searches
                actual_season = parse_result.season_number
                actual_episodes = parse_result.episode_numbers
378 379
            else:
                if not parse_result.is_air_by_date:
380
                    sickrage.app.log.debug(
echel0n's avatar
echel0n committed
381
                        "This is supposed to be a date search but the result " + result.name + " didn't parse as one, skipping it")
382
                    continue
383 384
                else:
                    airdate = parse_result.air_date.toordinal()
385 386
                    dbData = [x for x in sickrage.app.main_db.get_many('tv_episodes', result.show.indexerid)
                              if x['airdate'] == airdate]
387

echel0n's avatar
echel0n committed
388
                    if len(dbData) != 1:
389
                        sickrage.app.log.warning(
echel0n's avatar
echel0n committed
390
                            "Tried to look up the date for the episode " + result.name + " but the database didn't give proper results, skipping it")
391
                        continue
392

echel0n's avatar
echel0n committed
393 394
                    actual_season = int(dbData[0]["season"])
                    actual_episodes = [int(dbData[0]["episode"])]
395 396

            # make sure we want the episode
397
            wantEp = False
398
            for epNo in actual_episodes:
399
                if result.show.want_episode(actual_season, epNo, result.quality, manualSearch, downCurQuality):
400
                    wantEp = True
401 402

            if not wantEp:
403
                sickrage.app.log.info(
echel0n's avatar
echel0n committed
404
                    "RESULT:[{}] QUALITY:[{}] IGNORED!".format(result.name, Quality.qualityStrings[result.quality]))
405 406 407
                continue

            # make a result object
408
            result.episodes = []
409
            for curEp in actual_episodes:
410
                result.episodes.append(result.show.get_episode(actual_season, curEp))
411

412
            sickrage.app.log.debug(
echel0n's avatar
echel0n committed
413
                "FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(result.name, Quality.qualityStrings[result.quality],
414
                                                                 result.url))
415

416 417
            if len(result.episodes) == 1:
                epNum = result.episodes[0].episode
418
                sickrage.app.log.debug("Single episode result.")
419
            elif len(result.episodes) > 1:
420
                epNum = MULTI_EP_RESULT
421
                sickrage.app.log.debug(
422
                    "Separating multi-episode result to check for later - result contains episodes: " + str(
423
                        parse_result.episode_numbers))
424
            elif len(result.episodes) == 0:
425
                epNum = SEASON_RESULT
426
                sickrage.app.log.debug("Separating full season result to check for later")
427 428 429 430

            if epNum not in results:
                results[epNum] = [result]
            else:
echel0n's avatar
echel0n committed
431
                results[epNum] += [result]
432 433 434

        return results

435 436 437 438
    def find_propers(self, episodes):
        results = []

        for episode in episodes:
439
            show = findCertainShow(int(episode["showid"]))
440 441
            if not show:
                continue
442

443
            ep_obj = show.get_episode(int(episode["season"]), int(episode["episode"]))
444 445 446 447 448
            for term in self.proper_strings:
                search_strngs = self._get_episode_search_strings(ep_obj, add_string=term)
                for item in self.search(search_strngs[0], ep_obj=ep_obj):
                    result = self.getResult([ep_obj])
                    result.name, result.url = self._get_title_and_url(item)
449 450 451
                    if not validate_url(result.url) and not result.url.startswith('magnet'):
                        continue

452 453 454 455 456 457 458
                    result.seeders, result.leechers = self._get_result_stats(item)
                    result.size = self._get_size(item)
                    result.date = datetime.datetime.today()
                    result.show = show
                    results.append(result)

        return results
459

460 461
    def add_cookies_from_ui(self):
        """
echel0n's avatar
echel0n committed
462 463
        Add the cookies configured from UI to the providers requests session.
        :return: dict
464 465
        """

466 467 468 469 470
        if isinstance(self, TorrentRssProvider) and not self.cookies:
            return {'result': True,
                    'message': 'This is a TorrentRss provider without any cookies provided. '
                               'Cookies for this provider are considered optional.'}

471
        # This is the generic attribute used to manually add cookies for provider authentication
echel0n's avatar
echel0n committed
472 473 474 475 476 477 478 479 480 481
        if not self.enable_cookies:
            return {'result': False,
                    'message': 'Adding cookies is not supported for provider: {}'.format(self.name)}

        if not self.cookies:
            return {'result': False,
                    'message': 'No Cookies added from ui for provider: {}'.format(self.name)}

        cookie_validator = re.compile(r'^([\w%]+=[\w%]+)(;[\w%]+=[\w%]+)*$')
        if not cookie_validator.match(self.cookies):
482
            sickrage.app.alerts.message(
echel0n's avatar
echel0n committed
483 484 485 486 487 488
                'Failed to validate cookie for provider {}'.format(self.name),
                'Cookie is not correctly formatted: {}'.format(self.cookies))

            return {'result': False,
                    'message': 'Cookie is not correctly formatted: {}'.format(self.cookies)}

489 490 491
        if hasattr(self, 'required_cookies') and not all(
                req_cookie in [x.rsplit('=', 1)[0] for x in self.cookies.split(';')] for req_cookie in
                self.required_cookies):
echel0n's avatar
echel0n committed
492 493 494 495 496 497
            return {'result': False,
                    'message': "You haven't configured the required cookies. Please login at {provider_url}, "
                               "and make sure you have copied the following cookies: {required_cookies!r}"
                        .format(provider_url=self.name, required_cookies=self.required_cookies)}

        # cookie_validator got at least one cookie key/value pair, let's return success
498
        add_dict_to_cookiejar(self.session.cookies, dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
echel0n's avatar
echel0n committed
499 500 501 502 503 504 505 506 507 508 509

        return {'result': True,
                'message': ''}

    def check_required_cookies(self):
        """
        Check if we have the required cookies in the requests sessions object.

        Meaning that we've already successfully authenticated once, and we don't need to go through this again.
        Note! This doesn't mean the cookies are correct!
        """
510 511 512 513 514 515
        if hasattr(self, 'required_cookies'):
            return all(dict_from_cookiejar(self.session.cookies).get(cookie) for cookie in self.required_cookies)

        # A reminder for the developer, implementing cookie based authentication.
        sickrage.app.log.error(
            'You need to configure the required_cookies attribute, for the provider: {}'.format(self.name))
echel0n's avatar
echel0n committed
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535

    def cookie_login(self, check_login_text, check_url=None):
        """
        Check the response for text that indicates a login prompt.

        In that case, the cookie authentication was not successful.
        :param check_login_text: A string that's visible when the authentication failed.
        :param check_url: The url to use to test the login with cookies. By default the providers home page is used.

        :return: False when authentication was not successful. True if successful.
        """
        check_url = check_url or self.urls['base_url']

        if self.check_required_cookies():
            # All required cookies have been found within the current session, we don't need to go through this again.
            return True

        if self.cookies:
            result = self.add_cookies_from_ui()
            if not result['result']:
536
                sickrage.app.alerts.message(result['message'])
537
                sickrage.app.log.warning(result['message'])
echel0n's avatar
echel0n committed
538 539
                return False
        else:
540
            sickrage.app.log.warning('Failed to login, you will need to add your cookies in the provider '
541
                                     'settings')
echel0n's avatar
echel0n committed
542

543
            sickrage.app.alerts.error(
echel0n's avatar
echel0n committed
544 545 546 547
                'Failed to auth with {provider}'.format(provider=self.name),
                'You will need to add your cookies in the provider settings')
            return False

echel0n's avatar
echel0n committed
548
        response = self.session.get(check_url)
echel0n's avatar
echel0n committed
549 550
        if any([not response, not (response.text and response.status_code == 200),
                check_login_text.lower() in response.text.lower()]):
551
            sickrage.app.log.warning('Please configure the required cookies for this provider. Check your '
552
                                     'provider settings')
echel0n's avatar
echel0n committed
553

554
            sickrage.app.alerts.error(
echel0n's avatar
echel0n committed
555 556 557
                'Wrong cookies for {}'.format(self.name),
                'Check your provider settings'
            )
echel0n's avatar
echel0n committed
558
            self.session.cookies.clear()
echel0n's avatar
echel0n committed
559 560 561
            return False
        else:
            return True
562

563 564
    @classmethod
    def getDefaultProviders(cls):
565
        pass
566 567 568

    @classmethod
    def getProvider(cls, name):
569
        providerMatch = [x for x in cls.getProviders() if x.name == name]
570 571 572
        if len(providerMatch) == 1:
            return providerMatch[0]

573 574
    @classmethod
    def getProviderByID(cls, id):
575
        providerMatch = [x for x in cls.getProviders() if x.id == id]
576 577 578
        if len(providerMatch) == 1:
            return providerMatch[0]

579
    @classmethod
580
    def getProviders(cls):
581 582
        modules = [TorrentProvider.type, NZBProvider.type]
        for type in []:
583
            modules += cls.loadProviders(type)
584 585 586
        return modules

    @classmethod
587
    def loadProviders(cls, type):
588 589
        providers = []
        pregex = re.compile('^([^_]*?)\.py$', re.IGNORECASE)
590
        path = os.path.join(os.path.dirname(__file__), type)
591
        names = [pregex.match(m) for m in os.listdir(path)]
592
        providers += [cls.loadProvider(name.group(1), type) for name in names if name]
593 594 595
        return providers

    @classmethod
596
    def loadProvider(cls, name, type, *args, **kwargs):
597
        import inspect
598
        members = dict(
599 600 601
            inspect.getmembers(
                importlib.import_module('.{}.{}'.format(type, name), 'sickrage.providers'),
                lambda x: hasattr(x, 'type') and x not in [NZBProvider, TorrentProvider])
602
        )
603
        return [v for v in members.values() if hasattr(v, 'type') and v.type == type][0](*args, **kwargs)
604

605

606
class TorrentProvider(GenericProvider):
607
    type = 'torrent'
608

609 610
    def __init__(self, name, url, private):
        super(TorrentProvider, self).__init__(name, url, private)
echel0n's avatar
echel0n committed
611
        self.ratio = None
612 613 614

    @property
    def isActive(self):
615
        return sickrage.app.config.use_torrents and self.isEnabled
616

617 618
    @property
    def imageName(self):
619
        return self.id
620

echel0n's avatar
echel0n committed
621 622 623 624 625 626 627 628
    @property
    def seed_ratio(self):
        """
        Provider should override this value if custom seed ratio enabled
        It should return the value of the provider seed ratio
        """
        return self.ratio

629
    def getResult(self, episodes=None):
630 631 632 633 634 635 636
        """
        Returns a result of the correct type for this provider
        """
        result = TorrentSearchResult(episodes)
        result.provider = self
        return result

echel0n's avatar
echel0n committed
637
    def get_content(self, url):
638
        result = None
echel0n's avatar
echel0n committed
639

640 641 642 643 644 645
        def verify_torrent(content):
            try:
                if bencode.bdecode(content).get('info'):
                    return content
            except Exception:
                pass
echel0n's avatar
echel0n committed
646 647

        if url.startswith('magnet'):
648
            # try iTorrents
echel0n's avatar
echel0n committed
649 650 651 652
            info_hash = str(re.findall(r'urn:btih:([\w]{32,40})', url)[0]).upper()
            if len(info_hash) == 32:
                info_hash = b16encode(b32decode(info_hash)).upper()

653
            if info_hash:
echel0n's avatar
echel0n committed
654
                torrent_url = "https://itorrents.org/torrent/{info_hash}.torrent".format(info_hash=info_hash)
655
                result = verify_torrent(super(TorrentProvider, self).get_content(torrent_url))
echel0n's avatar
echel0n committed
656

657 658 659 660 661 662
                try:
                    # add to external api database
                    TorrentCacheAPI().add(url)
                    result = verify_torrent(b64decode(TorrentCacheAPI().get(info_hash)['data']['content']).strip())
                except Exception:
                    pass
echel0n's avatar
echel0n committed
663

664 665
        if not result:
            result = verify_torrent(super(TorrentProvider, self).get_content(url))
echel0n's avatar
echel0n committed
666

667
        return result
echel0n's avatar
echel0n committed
668

669
    def _get_title_and_url(self, item):
670
        title, download_url = '', ''
671 672
        if isinstance(item, (dict, FeedParserDict)):
            title = item.get('title', '')
673
            download_url = item.get('url', item.get('link', ''))
674 675 676 677 678
        elif isinstance(item, (list, tuple)) and len(item) > 1:
            title = item[0]
            download_url = item[1]

        # Temp global block `DIAMOND` releases
679
        if title.endswith('DIAMOND'):
680
            sickrage.app.log.info('Skipping DIAMOND release for mass fake releases.')
681
            title = download_url = 'FAKERELEASE'
682
        else:
683
            title = self._clean_title_from_provider(title)
684 685

        download_url = download_url.replace('&amp;', '&')
686 687 688

        return title, download_url

689
    def _get_size(self, item):
690
        return item.get('size', -1)
691

echel0n's avatar
echel0n committed
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
    def download_result(self, result):
        """
        Downloads a result to the appropriate black hole folder.

        :param result: SearchResult instance to download.
        :return: boolean, True on success
        """

        if not result.content:
            return False

        filename = self.make_filename(result.name)

        sickrage.app.log.info("Saving TORRENT to " + filename)

        # write content to torrent file
        with io.open(filename, 'wb') as f:
            f.write(result.content)

        return True

713 714 715 716
    @staticmethod
    def _clean_title_from_provider(title):
        return (title or '').replace(' ', '.')

717
    def make_filename(self, name):
echel0n's avatar
echel0n committed
718
        return os.path.join(sickrage.app.config.torrent_dir, '{}.torrent'.format(sanitizeFileName(name)))
719

720 721 722
    def add_trackers(self, result):
        """
        Adds public trackers to either torrent file or magnet link
723 724
        :param result: SearchResult
        :return: SearchResult
725 726 727
        """

        try:
echel0n's avatar
echel0n committed
728
            trackers_list = self.session.get('https://cdn.sickrage.ca/torrent_trackers/').text.split()
729 730 731 732
        except Exception:
            trackers_list = []

        if trackers_list:
733
            # adds public torrent trackers to magnet url
734
            if result.url.startswith('magnet:'):
735 736
                if not result.url.endswith('&tr='):
                    result.url += '&tr='
737
                result.url += '&tr='.join(trackers_list)
738 739 740

            # adds public torrent trackers to content
            if result.content:
741
                decoded_data = bencode.bdecode(result.content)
742 743 744
                if not decoded_data.get('announce-list'):
                    decoded_data[b'announce-list'] = []

745 746 747 748 749 750 751
                for tracker in trackers_list:
                    if tracker not in decoded_data['announce-list']:
                        decoded_data['announce-list'].append([str(tracker)])
                result.content = bencode.bencode(decoded_data)

        return result

752
    @classmethod
753
    def getProviders(cls):
754
        return super(TorrentProvider, cls).loadProviders(cls.type)
755 756 757


class NZBProvider(GenericProvider):
758
    type = 'nzb'
759

760 761
    def __init__(self, name, url, private):
        super(NZBProvider, self).__init__(name, url, private)
echel0n's avatar
echel0n committed
762 763
        self.api_key = ''
        self.username = ''
764
        self.torznab = False
765 766 767

    @property
    def isActive(self):
768
        return sickrage.app.config.use_nzbs and self.isEnabled
769

770 771
    @property
    def imageName(self):
772
        return self.id
773

774
    def getResult(self, episodes=None):
775 776 777 778
        """
        Returns a result of the correct type for this provider
        """
        result = NZBSearchResult(episodes)
779
        result.resultType = ('nzb', 'torznab')[self.torznab]
780 781 782
        result.provider = self
        return result

783
    def _get_size(self, item):
784
        return item.get('size', -1)
785

echel0n's avatar
echel0n committed
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
    def download_result(self, result):
        """
        Downloads a result to the appropriate black hole folder.

        :param result: SearchResult instance to download.
        :return: boolean, True on success
        """

        if not result.content:
            return False

        filename = self.make_filename(result.name)

        # Support for Jackett/TorzNab
        if (result.url.endswith('torrent') or result.url.startswith('magnet')) and self.type in ['nzb', 'newznab']:
            filename = filename.rsplit('.', 1)[0] + '.' + 'torrent'

        if result.resultType == "nzb":
            sickrage.app.log.info("Saving NZB to " + filename)

            # write content to torrent file
            with io.open(filename, 'wb') as f:
                f.write(result.content)

            return True
        elif result.resultType == "nzbdata":
            filename = os.path.join(sickrage.app.config.nzb_dir, result.name + ".nzb")

            sickrage.app.log.info("Saving NZB to " + filename)

            # save the data to disk
            try:
                with io.open(filename, 'w') as fileOut:
                    fileOut.write(result.extraInfo[0])

821
                chmod_as_parent(filename)
echel0n's avatar
echel0n committed
822 823 824

                return True
            except EnvironmentError as e:
echel0n's avatar
echel0n committed
825
                sickrage.app.log.error("Error trying to save NZB to black hole: {}".format(e))
echel0n's avatar
echel0n committed
826

827
    def make_filename(self, name):
echel0n's avatar
echel0n committed
828
        return os.path.join(sickrage.app.config.nzb_dir, '{}.nzb'.format(sanitizeFileName(name)))
829

830
    @classmethod
831
    def getProviders(cls):
832
        return super(NZBProvider, cls).loadProviders(cls.type)
833 834 835


class TorrentRssProvider(TorrentProvider):
836
    type = 'torrentrss'
837 838 839 840 841 842 843 844 845

    def __init__(self,
                 name,
                 url,
                 cookies='',
                 titleTAG='title',
                 search_mode='eponly',
                 search_fallback=False,
                 enable_daily=False,
846
                 enable_backlog=False,
847
                 default=False, ):
848
        super(TorrentRssProvider, self).__init__(name, clean_url(url), False)
849 850

        self.cache = TorrentRssCache(self)
echel0n's avatar
echel0n committed
851
        self.supports_backlog = False
852 853 854 855 856

        self.search_mode = search_mode
        self.search_fallback = search_fallback
        self.enable_daily = enable_daily
        self.enable_backlog = enable_backlog
857
        self.enable_cookies = True
858
        self.cookies = cookies
859
        self.required_cookies = ('uid', 'pass')
860
        self.titleTAG = titleTAG
861
        self.default = default
862 863 864

    def _get_title_and_url(self, item):

865 866
        title = item.get(self.titleTAG, '')
        title = self._clean_title_from_provider(title)
867 868 869 870 871 872 873

        attempt_list = [lambda: item.get('torrent_magneturi'),

                        lambda: item.enclosures[0].href,

                        lambda: item.get('link')]

874
        url = ''
875 876 877 878 879 880 881 882 883 884 885 886
        for cur_attempt in attempt_list:
            try:
                url = cur_attempt()
            except Exception:
                continue

            if title and url:
                break

        return title, url

    def validateRSS(self):
echel0n's avatar
echel0n committed
887 888
        torrent_file = None

889
        try:
echel0n's avatar
echel0n committed
890 891 892
            add_cookie = self.add_cookies_from_ui()
            if not add_cookie.get('result'):
                return add_cookie
893

894
            data = self.cache._get_rss_data()['entries']
895
            if not data:
echel0n's avatar
echel0n committed
896 897
                return {'result': False,
                        'message': 'No items found in the RSS feed {}'.format(self.urls['base_url'])}
898 899 900 901

            (title, url) = self._get_title_and_url(data[0])

            if not title:
echel0n's avatar
echel0n committed
902 903
                return {'result': False,
                        'message': 'Unable to get title from first item'}
904 905

            if not url:
echel0n's avatar
echel0n committed
906 907
                return {'result': False,
                        'message': 'Unable to get torrent url from first item'}
908 909

            if url.startswith('magnet:') and re.search(r'urn:btih:([\w]{32,40})', url):
echel0n's avatar
echel0n committed
910 911
                return {'result': True,
                        'message': 'RSS feed Parsed correctly'}
912 913
            else:
                try:
echel0n's avatar
echel0n committed
914
                    torrent_file = self.session.get(url).content
915 916
                    bencode.bdecode(torrent_file)
                except Exception as e:
echel0n's avatar
echel0n committed
917 918
                    if data:
                        self.dumpHTML(torrent_file)
echel0n's avatar
echel0n committed
919
                    return {'result': False,
echel0n's avatar
echel0n committed
920
                            'message': 'Torrent link is not a valid torrent file: {}'.format(e)}
921

echel0n's avatar
echel0n committed
922 923
            return {'result': True,
                    'message': 'RSS feed Parsed correctly'}
924 925

        except Exception as e:
echel0n's avatar
echel0n committed
926
            return {'result': False,
echel0n's avatar
echel0n committed
927
                    'message': 'Error when trying to load RSS: {}'.format(e)}
928 929 930

    @staticmethod
    def dumpHTML(data):
931
        dumpName = os.path.join(sickrage.app.cache_dir, 'custom_torrent.html')
932 933 934 935

        try:
            with io.open(dumpName, 'wb') as fileOut:
                fileOut.write(data)
echel0n's avatar
echel0n committed
936

937
            chmod_as_parent(dumpName)
echel0n's avatar
echel0n committed
938

939
            sickrage.app.log.info("Saved custom_torrent html dump %s " % dumpName)
940
        except IOError as e:
941
            sickrage.app.log.error("Unable to save the file: %s " % repr(e))
942
            return False
echel0n's avatar
echel0n committed
943

944 945 946
        return True

    @classmethod
947
    def getProviders(cls):
echel0n's avatar
echel0n committed
948 949 950
        providers = cls.getDefaultProviders()

        try:
951
            for curProviderStr in sickrage.app.config.custom_providers.split('!!!'):
echel0n's avatar
echel0n committed
952 953 954 955 956 957 958
                if not len(curProviderStr):
                    continue

                try:
                    cur_type, curProviderData = curProviderStr.split('|', 1)
                    if cur_type == "torrentrss":
                        cur_name, cur_url, cur_cookies, cur_title_tag = curProviderData.split('|')
959
                        providers += [TorrentRssProvider(cur_name, cur_url, cur_cookies, cur_title_tag)]
echel0n's avatar
echel0n committed
960 961 962 963 964 965
                except Exception:
                    continue
        except Exception:
            pass

        return providers
966 967

    @classmethod
968 969
    def getDefaultProviders(cls):
        return [
970
            cls('showRSS', 'showrss.info', '', 'title', 'eponly', False, False, False, True)
971
        ]
972 973 974


class NewznabProvider(NZBProvider):
975
    type = 'newznab'
976

977 978 979
    def __init__(self, name, url, key='0', catIDs='5030,5040', search_mode='eponly', search_fallback=False,
                 enable_daily=False, enable_backlog=False, default=False):
        super(NewznabProvider, self).__init__(name, clean_url(url), bool(key != '0'))
980

echel0n's avatar
echel0n committed
981 982
        self.key = key

983 984 985 986
        self.search_mode = search_mode
        self.search_fallback = search_fallback
        self.enable_daily = enable_daily
        self.enable_backlog = enable_backlog
echel0n's avatar
echel0n committed
987

988 989
        self.catIDs = catIDs
        self.default = default
echel0n's avatar
echel0n committed
990

991 992 993 994
        self.caps = False
        self.cap_tv_search = None
        self.force_query = False

echel0n's avatar
echel0n committed
995
        self.cache = TVCache(self, min_time=30)
996

997
    def set_caps(self, data):
998
        """
999
        Set caps.
1000
        """
1001 1002
        if not data:
            return
1003

1004 1005
        def _parse_cap(tag):
            elm = data.find(tag)
1006 1007
            is_supported = elm and all([elm.get('supportedparams'), elm.get('available') == 'yes'])
            return elm['supportedparams'].split(',') if is_supported else []
1008

1009
        self.cap_tv_search = _parse_cap('tv-search')
1010

1011
        self.caps = any(self.cap_tv_search)
1012

1013 1014 1015
    def get_newznab_categories(self, just_caps=False):
        """
        Use the newznab provider url and apikey to get the capabilities.
1016

1017 1018 1019 1020 1021
        Makes use of the default newznab caps param. e.a. http://yournewznab/api?t=caps&apikey=skdfiw7823sdkdsfjsfk
        Returns a tuple with (succes or not, array with dicts [{'id': '5070', 'name': 'Anime'},
        {'id': '5080', 'name': 'Documentary'}, {'id': '5020', 'name': 'Foreign'}...etc}], error message)
        """
        return_categories = []
1022

1023 1024
        if not self._check_auth():
            return False, return_categories, 'Provider requires auth and your key is not set'
1025

1026 1027 1028
        url_params = {'t': 'caps'}
        if self.private and self.key:
            url_params['apikey'] = self.key
1029

1030
        try:
echel0n's avatar
echel0n committed
1031
            response = self.session.get(urljoin(self.urls['base_url'], 'api'), params=url_params).text
1032 1033
        except Exception:
            error_string = 'Error getting caps xml for [{}]'.format(self.name)
1034
            sickrage.app.log.warning(error_string)
1035
            return False, return_categories, error_string
1036

1037 1038 1039
        with bs4_parser(response) as html:
            if not html.find('categories'):
                error_string = 'Error parsing caps xml for [{}]'.format(self.name)
1040
                sickrage.app.log.debug(error_string)
1041
                return False, return_categories, error_string
1042

1043 1044 1045
            self.set_caps(html.find('searching'))
            if just_caps:
                return
1046

1047 1048 1049 1050 1051 1052
            for category in html('category'):
                if 'TV' in category.get('name', '') and category.get('id', ''):
                    return_categories.append({'id': category['id'], 'name': category['name']})
                    for subcat in category('subcat'):
                        if subcat.get('name', '') and subcat.get('id', ''):
                            return_categories.append({'id': subcat['id'], 'name': subcat['name']})
1053

1054
            return True, return_categories, ''
1055 1056

    def _doGeneralSearch(self, search_string):
1057
        return self.search({'q': search_string})
1058

1059
    def _check_auth(self):
1060
        if self.private and not self.key:
1061
            sickrage.app.log.warning('Invalid api key for {}. Check your settings'.format(self.name))
echel0n's avatar
echel0n committed
1062 1063
            return False

1064 1065
        return True

1066
    def _check_auth_from_data(self, data):
1067
        """
1068 1069 1070
        Check that the returned data is valid.

        :return: _check_auth if valid otherwise False if there is an error
1071
        """
1072
        if data('categories') + data('item'):
1073
            return self._check_auth()
1074 1075

        try:
1076 1077 1078 1079 1080
            err_desc = data.error.attrs['description']
            if not err_desc:
                raise Exception
        except (AttributeError, TypeError):
            return self._check_auth()
1081

1082
        sickrage.app.log.info(err_desc)
1083

1084
        return False
1085

echel0n's avatar
echel0n committed
1086
    def search(self, search_strings, age=0, ep_obj=None, **kwargs):
1087 1088 1089 1090 1091
        """
        Search indexer using the params in search_strings, either for latest releases, or a string/id search.

        :return: list of results in dict form
        """
echel0n's avatar
echel0n committed
1092
        results = []
1093

1094
        if not self._check_auth():
echel0n's avatar
echel0n committed
1095
            return results
1096

1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
        # For providers that don't have caps, or for which the t=caps is not working.
        if not self.caps:
            self.get_newznab_categories(just_caps=True)

        for mode in search_strings:
            self.torznab = False
            search_params = {
                't': 'search',
                'limit': 100,
                'offset': 0,
                'cat': self.catIDs.strip(', ') or '5030,5040',