fake user agent for IMDB scraping

python/lib/tmdbscraper/api_utils.py

@@ -46,6 +46,7 @@
 
 
 def set_headers(headers):
+    HEADERS.clear()
     HEADERS.update(headers)
 
 
@@ -65,6 +66,8 @@ def load_info(url, params=None, default=None, resp_type = 'json'):
         url = url + '?' + urlencode(params)
     if xbmc:
         xbmc.log('Calling URL "{}"'.format(url), xbmc.LOGDEBUG)
+    if HEADERS:
+        xbmc.log(str(HEADERS), xbmc.LOGDEBUG)
     req = Request(url, headers=HEADERS)
     try:
         response = urlopen(req)

python/lib/tmdbscraper/fanarttv.py

@@ -19,6 +19,12 @@
     'movieposter': 'poster'
 }
 
+HEADERS = (
+    ('User-Agent', 'Kodi Movie scraper by Team Kodi'),
+    ('api-key', API_KEY),
+)
+
+
 def get_details(uniqueids, clientkey, language, set_tmdbid):
     media_id = _get_mediaid(uniqueids)
     if not media_id:
@@ -48,7 +54,7 @@ def _get_mediaid(uniqueids):
             return uniqueids[source]
 
 def _get_data(media_id, clientkey):
-    headers = {'api-key': API_KEY}
+    headers = dict(HEADERS)
     if clientkey:
         headers['client-key'] = clientkey
     api_utils.set_headers(headers)

python/lib/tmdbscraper/imdbratings.py

@@ -32,6 +32,11 @@
 IMDB_VOTES_REGEX_PREVIOUS = re.compile(r'itemprop="ratingCount".*?>.*?([\d,]+).*?<')
 IMDB_TOP250_REGEX_PREVIOUS = re.compile(r'Top Rated Movies #(\d+)')
 
+HEADERS = (
+    ('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'),
+    ('Accept', 'application/json'),
+)
+
 def get_details(uniqueids):
     imdb_id = get_imdb_id(uniqueids)
     if not imdb_id:
@@ -40,6 +45,7 @@ def get_details(uniqueids):
     return _assemble_imdb_result(votes, rating, top250)
 
 def _get_ratinginfo(imdb_id):
+    api_utils.set_headers(dict(HEADERS))
     response = api_utils.load_info(IMDB_RATINGS_URL.format(imdb_id), default = '', resp_type='text')
     return _parse_imdb_result(response)
 

python/lib/tmdbscraper/tmdbapi.py

@@ -31,7 +31,6 @@
     ('User-Agent', 'Kodi Movie scraper by Team Kodi'),
     ('Accept', 'application/json'),
 )
-api_utils.set_headers(dict(HEADERS))
 
 TMDB_PARAMS = {'api_key': 'f090bb54758cabf231fb605d3e3e0468'}
 BASE_URL = 'https://api.themoviedb.org/3/{}'
@@ -61,6 +60,7 @@ def search_movie(query, year=None, language=None, page=None):
         params['page'] = page
     if year is not None:
         params['year'] = str(year)
+    api_utils.set_headers(dict(HEADERS))
     return api_utils.load_info(theurl, params=params)
 
 
@@ -77,6 +77,7 @@ def find_movie_by_external_id(external_id, language=None):
     theurl = FIND_URL.format(external_id)
     params = _set_params(None, language)
     params['external_source'] = 'imdb_id'
+    api_utils.set_headers(dict(HEADERS))
     return api_utils.load_info(theurl, params=params)
 
 
@@ -93,6 +94,7 @@ def get_movie(mid, language=None, append_to_response=None):
     """
     xbmc.log('using movie id of %s to get movie details' % mid, xbmc.LOGDEBUG)
     theurl = MOVIE_URL.format(mid)
+    api_utils.set_headers(dict(HEADERS))
     return api_utils.load_info(theurl, params=_set_params(append_to_response, language))
 
 
@@ -108,6 +110,7 @@ def get_collection(collection_id, language=None, append_to_response=None):
     """
     xbmc.log('using collection id of %s to get collection details' % collection_id, xbmc.LOGDEBUG)
     theurl = COLLECTION_URL.format(collection_id)
+    api_utils.set_headers(dict(HEADERS))
     return api_utils.load_info(theurl, params=_set_params(append_to_response, language))
 
 
@@ -119,6 +122,7 @@ def get_configuration():
     :return: configuration details or error
     """
     xbmc.log('getting configuration details', xbmc.LOGDEBUG)
+    api_utils.set_headers(dict(HEADERS))
     return api_utils.load_info(CONFIG_URL, params=TMDB_PARAMS.copy())
 
 

python/lib/tmdbscraper/traktratings.py

@@ -36,7 +36,6 @@
     ('trakt-api-version', '2'),
     ('Content-Type', 'application/json'),
 )
-api_utils.set_headers(dict(HEADERS))
 
 MOVIE_URL = 'https://api.trakt.tv/movies/{}'
 
@@ -46,6 +45,7 @@ def get_trakt_ratinginfo(uniqueids):
     result = {}
     url = MOVIE_URL.format(imdb_id)
     params = {'extended': 'full'}
+    api_utils.set_headers(dict(HEADERS))
     movie_info = api_utils.load_info(url, params=params, default={})
     if(movie_info):
         if 'votes' in movie_info and 'rating' in movie_info: