Fixed issues with grabbing redirect url for session POST method.

Fixed login issue for FileList.
Fixed issue with parsing torrents from Zooqle and YGGTorrent.

changelog.md

 # Changelog
 
-- * b881d12 - 2018-11-17: Pre-Release v9.4.36.dev3 
+- * 5d32bc5 - 2018-11-17: Fixed issues with grabbing redirect url for session POST method. Fixed login issue for FileList. Fixed issue with parsing torrents from Zooqle and YGGTorrent. 
+- * 5acf339 - 2018-11-17: Pre-Release v9.4.36.dev3 
 - * 34dd950 - 2018-11-17: Fixed issue #322 - NoneType error when trying to view Trakt `most anticipated` shows due to indexer failing to have artwork available. Refactored error page to new UI look. 
 - * 3299a1e - 2018-11-17: Pre-Release v9.4.36.dev2 
 - * 9048cd4 - 2018-11-17: Refactored DanishBits torrent provider. 

sickrage/core/helpers/__init__.py

@@ -548,8 +548,9 @@ def move_and_symlink_file(src_file, dest_file):
  sickrage.app.log.warning('Failed to create symlink of {src} at {dest}. Error: {error!r}'.format(
  **{'src': src_file, 'dest': dest_file, 'error': e}))
  else:
- sickrage.app.log.warning("Failed to create symlink of {src} at {dest}. Error: {error!r}. Copying instead".format(
- **{'src': src_file, 'dest': dest_file, 'error': e}))
+ sickrage.app.log.warning(
+ "Failed to create symlink of {src} at {dest}. Error: {error!r}. Copying instead".format(
+ **{'src': src_file, 'dest': dest_file, 'error': e}))
  copy_file(src_file, dest_file)
 
 
@@ -1059,8 +1060,8 @@ def restoreSR(srcDir, dstDir):
  if os.path.exists(os.path.join(dstDir, 'database')):
  move_file(os.path.join(dstDir, 'database'), os.path.join(dstDir, '{}.bak-{}'
  .format('database',
- datetime.datetime.now().strftime(
- '%Y%m%d_%H%M%S'))))
+  datetime.datetime.now().strftime(
+  '%Y%m%d_%H%M%S'))))
  move_file(os.path.join(srcDir, 'database'), dstDir)
 
  # cache
@@ -1068,8 +1069,8 @@ def restoreSR(srcDir, dstDir):
  if os.path.exists(os.path.join(dstDir, 'cache')):
  move_file(os.path.join(dstDir, 'cache'), os.path.join(dstDir, '{}.bak-{}'
  .format('cache',
- datetime.datetime.now().strftime(
- '%Y%m%d_%H%M%S'))))
+   datetime.datetime.now().strftime(
+  '%Y%m%d_%H%M%S'))))
  move_file(os.path.join(srcDir, 'cache'), dstDir)
 
  return True

sickrage/core/websession/__init__.py

@@ -62,7 +62,7 @@ class WebSession(Session):
  self.cloudflare = cloudflare
 
  # add hooks
- self.hooks['response'] += [WebHooks.log_url, WebHooks.redirect_posts]
+ self.hooks['response'] += [WebHooks.log_url]
 
  @staticmethod
  def _get_ssl_cert(verify):
@@ -74,11 +74,17 @@ class WebSession(Session):
  """
  return certifi.where() if all([sickrage.app.config.ssl_verify, verify]) else False
 
- def request(self, method, url, verify=False, random_ua=False, *args, **kwargs):
+ def request(self, method, url, verify=False, random_ua=False, allow_post_redirects=False, *args, **kwargs):
  self.headers.update({'Accept-Encoding': 'gzip, deflate',
  'User-Agent': (sickrage.app.user_agent, UserAgent().random)[random_ua]})
 
- if not verify: disable_warnings()
+ if not verify:
+ disable_warnings()
+
+ if allow_post_redirects and method == 'POST':
+ sickrage.app.log.debug('Retrieving redirect URL for {url}'.format(**{'url': url}))
+ response = super(WebSession, self).request(method, url, allow_redirects=False)
+  url = self.get_redirect_target(response) or url
 
  response = super(WebSession, self).request(method, url, verify=self._get_ssl_cert(verify), *args, **kwargs)
  if self.cloudflare:
@@ -154,34 +160,6 @@ class WebHooks(object):
  else:
  sickrage.app.log.debug('With post data: {}'.format(to_unicode(request.body)))
 
- @staticmethod
- def redirect_posts(resp, **kwargs):
- """Response hook to handle post method for URLs that are redirected"""
-
- if resp.request.method == 'POST' and resp.headers.get('Location'):
- # Get the session used or create a new one
- session = getattr(resp, 'session', requests.Session())
-
- # Get the original request
- original_request = resp.request
-
- # Set new URL
- location = resp.headers['Location']
- sickrage.app.log.debug('URL redirection detected for {} to {}'.format(original_request.url, location))
- original_request.url = location
-
- # Remove hooks from original request
- original_hooks = original_request.hooks
- original_request.hooks = session.hooks
-
- # Resend the request
- resp = session.send(original_request, **kwargs)
-
- # Add original hooks back to original request
- resp.hooks = original_hooks
-
- return resp
-
 
 class WebHelpers(object):
  @staticmethod

sickrage/providers/torrent/filelist.py

@@ -19,8 +19,6 @@
 
 from __future__ import unicode_literals
 
-import re
-
 from requests.compat import urljoin
 from requests.utils import dict_from_cookiejar
 
@@ -33,6 +31,7 @@ from sickrage.providers import TorrentProvider
 class FileListProvider(TorrentProvider):
  def __init__(self):
  super(FileListProvider, self).__init__('FileList', 'https://filelist.ro', True)
+
  # Credentials
  self.username = None
  self.password = None
@@ -59,19 +58,20 @@ class FileListProvider(TorrentProvider):
 
  login_params = {
  "username": self.username,
- "password": self.password
+ "password": self.password,
+ "ssl": 'yes'
  }
 
  try:
  response = self.session.post(self.urls["login"], data=login_params).text
  except Exception:
  sickrage.app.log.warning("Unable to connect to provider")
+ self.session.cookies.clear()
  return False
 
- if re.search("Invalid Username/password", response) \
- or re.search("<title>Login :: FileList.ro</title>", response) \
- or re.search("Login esuat!", response):
+ if 'logout.php' not in response:
  sickrage.app.log.warning("Invalid username or password. Check your settings")
+ self.session.cookies.clear()
  return False
 
  return True
@@ -94,12 +94,10 @@ class FileListProvider(TorrentProvider):
  for search_string in search_strings[mode]:
  if mode != "RSS":
  sickrage.app.log.debug("Search string: {}".format(search_string))
-
- search_params["search"] = search_string
- search_url = self.urls["search"]
+ search_params["search"] = search_string
 
  try:
- data = self.session.get(search_url, params=search_params).text
+ data = self.session.get(self.urls['search'], params=search_params).text
  results += self.parse(data, mode)
  except Exception:
  sickrage.app.log.debug("No data returned from provider")
@@ -116,7 +114,7 @@ class FileListProvider(TorrentProvider):
 
  results = []
 
- with bs4_parser(data, "html5lib") as html:
+ with bs4_parser(data) as html:
  torrent_rows = html.find_all("div", class_="torrentrow")
 
  # Continue only if at least one Release is found
@@ -164,9 +162,13 @@ class FileListProvider(TorrentProvider):
  torrent_size = cells[labels.index("Size")].find("span").get_text(strip=True)
  size = convert_size(torrent_size, -1)
 
- results += [
- {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers}
- ]
+ results += [{
+ 'title': title,
+ 'link': download_url,
+ 'size': size,
+ 'seeders': seeders,
+ 'leechers': leechers
+ }]
 
  if mode != "RSS":
  sickrage.app.log.debug("Found result: {}".format(title))

sickrage/providers/torrent/torrent9.py

-# coding=utf-8
+ # coding=utf-8
 # Author: Ludovic Reenaers <[email protected]>
 #
 # URL: https://sickrage.ca
@@ -27,9 +27,18 @@ import sickrage
 from sickrage.core.caches.tv_cache import TVCache
 from sickrage.core.helpers import bs4_parser, try_int, convert_size, validate_url
 from sickrage.providers import TorrentProvider
+ from __future__ import print_function, unicode_literals
 
+ import re
+ from urlparse import urljoin
 
-class Torrent9Provider(TorrentProvider):
+ import sickrage
+ from sickrage.core.caches.tv_cache import TVCache
+ from sickrage.core.helpers import bs4_parser, try_int, convert_size, validate_url
+ from sickrage.providers import TorrentProvider
+
+
+ class Torrent9Provider(TorrentProvider):
  def __init__(self):
  super(Torrent9Provider, self).__init__('Torrent9', 'ww1.torrent9.ph', False)
 

sickrage/providers/torrent/yggtorrent.py

@@ -31,7 +31,7 @@ from sickrage.providers import TorrentProvider
 class YggtorrentProvider(TorrentProvider):
  def __init__(self):
  """Initialize the class."""
- super(YggtorrentProvider, self).__init__('Yggtorrent', 'https://yggtorrent.to', True)
+ super(YggtorrentProvider, self).__init__('Yggtorrent', 'https://www.yggtorrent.to', True)
 
  # URLs
  self.urls.update({
@@ -156,7 +156,7 @@ class YggtorrentProvider(TorrentProvider):
 
  if not self._is_authenticated():
  try:
- self.session.post(self.urls['login'], data=login_params)
+ self.session.post(self.urls['login'], data=login_params, allow_post_redirects=True)
  except Exception:
  sickrage.app.log.warning('Unable to connect or login to provider')
  return False
@@ -169,10 +169,7 @@ class YggtorrentProvider(TorrentProvider):
 
  def _is_authenticated(self):
  try:
- response = self.session.get(self.urls['auth']).json()
+ self.session.get(self.urls['auth']).json()
  except Exception:
  return False
  return True
-
-
-provider = YggtorrentProvider()

sickrage/providers/torrent/zooqle.py

@@ -163,8 +163,8 @@ class ZooqleProvider(TorrentProvider):
 
  for item in data['entries']:
  try:
- title = item['title']
- download_url = item['torrent_magneturi']
+ title = item.get('title')
+ download_url = item.get('torrent_magneturi')
  if not all([title, download_url]):
  continue
 
@@ -172,9 +172,13 @@ class ZooqleProvider(TorrentProvider):
  leechers = try_int(item['torrent_peers'])
  size = try_int(item['torrent_contentlength'], -1)
 
- results += [
- {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers}
- ]
+ results += [{
+ 'title': title,
+ 'link': download_url,
+ 'size': size,
+ 'seeders': seeders,
+ 'leechers': leechers
+ }]
 
  if mode != 'RSS':
  sickrage.app.log.debug('Found result: {}'.format(title))