From ed50a05773802e783b00ad81d8091c79052b3247 Mon Sep 17 00:00:00 2001 From: dimkroon <111366411+dimkroon@users.noreply.github.com> Date: Wed, 6 Dec 2023 01:39:02 +0100 Subject: [PATCH] [plugin.video.viwx] v1.2.0 --- plugin.video.viwx/addon.xml | 27 +- plugin.video.viwx/changelog.txt | 20 ++ plugin.video.viwx/resources/lib/cache.py | 3 + plugin.video.viwx/resources/lib/fetch.py | 27 +- plugin.video.viwx/resources/lib/itv.py | 71 ++--- .../resources/lib/itv_account.py | 102 +++++-- plugin.video.viwx/resources/lib/itvx.py | 263 ++++++++++++++---- plugin.video.viwx/resources/lib/kodi_utils.py | 3 +- plugin.video.viwx/resources/lib/main.py | 207 ++++++++++---- plugin.video.viwx/resources/lib/parsex.py | 184 ++++++++++-- plugin.video.viwx/resources/lib/settings.py | 9 + plugin.video.viwx/resources/lib/xprogress.py | 253 +++++++++++++++++ plugin.video.viwx/resources/settings.xml | 2 +- 13 files changed, 949 insertions(+), 222 deletions(-) create mode 100644 plugin.video.viwx/resources/lib/xprogress.py diff --git a/plugin.video.viwx/addon.xml b/plugin.video.viwx/addon.xml index f1b0d3652..33ffcc062 100644 --- a/plugin.video.viwx/addon.xml +++ b/plugin.video.viwx/addon.xml @@ -1,5 +1,5 @@ - + @@ -30,15 +30,24 @@ resources/fanart.png -[B]v1.1.1[/B] -- fix: playing VOD on kodi Omega failed with error. +[B]v1.2.0[/B] +[B]Fixes:[/B] +* All categories failed to open due to various changes at ITVX. +* Error on opening some series named 'Other Episodes', due to changes at ITVX. +* Some sub-collections failed with error 'Not Found', due to error in ITVX data' +* Added support for hero and collections items of type 'page'. Fixes some collections being empty, or hero item not shown. -[B]v1.1.0[/B] -- Updated user-agent string to Firefox 118. -- Added support for Live TV items in collections. Fixes: collection 'ITVX Live Channels' is empty. -- Added support for shortFromSlider in collection, like the folder with short news-like items in the collection 'Rugby World Cup'. -- Added support for shorForm collections. Fixes: absence of the collection 'Rugby World Cup 2023'. -- Adapt to a change at itvx causing an empty 'Kids Collection'. +[B]New features:[/B] +* Added a 'My itvX' entry in the main menu with: + - My List - ITVX's My List. + - Continue Watching: supports continue watching on different devices and different platforms. + - Because You Watched: recommendations by ITV based on a recently watched programme. + - Recommended: general recommendations by ITV. +* All programmes and series now have a context menu item to add/remove the programme to/from My List. + +[B]Changes:[/B] +* When not signed in, a user is now always offered to sign in via viwX's settings when an item was opened that required authentication. +* Search now respects setting 'Hide premium content'. true diff --git a/plugin.video.viwx/changelog.txt b/plugin.video.viwx/changelog.txt index cc2c16fe5..160ad3fab 100644 --- a/plugin.video.viwx/changelog.txt +++ b/plugin.video.viwx/changelog.txt @@ -1,3 +1,23 @@ +v1.2.0 +Fixes: +- All categories failed to open due to various changes at ITVX. +- Error on opening some series named 'Other Episodes', due to changes at ITVX. +- Some sub-collections failed with error 'Not Found', due to error in ITVX data. +- Added support for hero and collections items of type 'page'. Fixes some collections being empty, or hero item not shown. + +New features: +- Added a 'My itvX' entry in the main menu with: + - My List - ITVX's My List. + - Continue Watching: supports continue watching on different devices and different platforms. + - Because You Watched: recommendations by ITV based on a recently watched programme. + - Recommended: general recommendations by ITV. +- All programmes and series now have a context menu item to add/remove the programme to/from My List. + +Changes: +- When not signed in, a user is now always offered to sign in via viwX's settings when an item was opened that required authentication. +- Search now respects setting 'Hide premium content'. +- Brushed up README thanks to JohnVeness. + v 1.1.1 - fix: playing VOD on kodi Omega failed with error. diff --git a/plugin.video.viwx/resources/lib/cache.py b/plugin.video.viwx/resources/lib/cache.py index b01c06b72..29b8508c1 100644 --- a/plugin.video.viwx/resources/lib/cache.py +++ b/plugin.video.viwx/resources/lib/cache.py @@ -27,6 +27,9 @@ __cache = {} +my_list_programmes = None + + def get_item(key): """Return the cached data if present in the cache and not expired. Return None otherwise. diff --git a/plugin.video.viwx/resources/lib/fetch.py b/plugin.video.viwx/resources/lib/fetch.py index 0c4affec9..87287b155 100644 --- a/plugin.video.viwx/resources/lib/fetch.py +++ b/plugin.video.viwx/resources/lib/fetch.py @@ -22,6 +22,7 @@ WEB_TIMEOUT = (3.5, 7) USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0' +USER_AGENT_VERSION = '118.0' logger = logging.getLogger('.'.join((logger_id, __name__.split('.', 2)[-1]))) @@ -132,14 +133,14 @@ def set_default_cookies(cookiejar: RequestsCookieJar = None): Return the cookiejar """ + s = requests.Session() + if isinstance(cookiejar, RequestsCookieJar): + s.cookies = cookiejar + elif cookiejar is not None: + raise ValueError("Parameter cookiejar must be an instance of RequestCookiejar") + # noinspection PyBroadException try: - s = requests.Session() - if isinstance(cookiejar, RequestsCookieJar): - s.cookies = cookiejar - elif cookiejar is not None: - raise ValueError("Parameter cookiejar must be an instance of RequestCookiejar") - # Make a request to reject all cookies. resp = s.get( 'https://identityservice.syrenis.com/Home/SaveConsent', @@ -260,6 +261,20 @@ def put_json(url, data, headers=None, **kwargs): return resp +def delete_json(url, data, headers=None, **kwargs): + """DELETE JSON data and return the response object or None if no data has been returned.""" + dflt_headers = {'Accept': 'application/json'} + if headers: + dflt_headers.update(headers) + resp = web_request('DELETE', url, dflt_headers, data, **kwargs) + if resp.status_code == 204: # No Content + return None + try: + return resp.json() + except json.JSONDecodeError: + raise FetchError(Script.localize(30920)) + + def get_document(url, headers=None, **kwargs): """GET any document. Expects the document to be UTF-8 encoded and returns the contents as string. diff --git a/plugin.video.viwx/resources/lib/itv.py b/plugin.video.viwx/resources/lib/itv.py index ce19b1975..c0a72be82 100644 --- a/plugin.video.viwx/resources/lib/itv.py +++ b/plugin.video.viwx/resources/lib/itv.py @@ -19,8 +19,6 @@ from . import fetch from . import kodi_utils -from .errors import AuthenticationError - logger = logging.getLogger(logger_id + '.itv') @@ -89,48 +87,33 @@ def get_live_schedule(hours=4, local_tz=None): } -def _request_stream_data(url, stream_type='live', retry_on_error=True): - from .itv_account import itv_session +def _request_stream_data(url, stream_type='live'): + from .itv_account import itv_session, fetch_authenticated session = itv_session() - try: - stream_req_data['user']['token'] = session.access_token - stream_req_data['client']['supportsAdPods'] = stream_type != 'live' - - if stream_type == 'live': - accept_type = 'application/vnd.itv.online.playlist.sim.v3+json' - # Live MUST have a featureset containing an item without outband-webvtt, or a bad request is returned. - min_features = ['mpeg-dash', 'widevine'] - else: - accept_type = 'application/vnd.itv.vod.playlist.v2+json' - # ITV appears now to use the min feature for catchup streams, causing subtitles - # to go missing if not specified here. Min and max both specifying webvtt appears to - # be no problem for catchup streams that don't have subtitles. - min_features = ['mpeg-dash', 'widevine', 'outband-webvtt', 'hd', 'single-track'] - - stream_req_data['variantAvailability']['featureset']['min'] = min_features - - stream_data = fetch.post_json( - url, stream_req_data, - headers={'Accept': accept_type}, - cookies=session.cookie) - - http_status = stream_data.get('StatusCode', 0) - if http_status == 401: - raise AuthenticationError - - return stream_data - except AuthenticationError: - if retry_on_error: - if session.refresh(): - return _request_stream_data(url, stream_type, retry_on_error=False) - else: - if kodi_utils.show_msg_not_logged_in(): - from xbmc import executebuiltin - executebuiltin('Addon.OpenSettings({})'.format(utils.addon_info.id)) - raise - else: - raise + stream_req_data['user']['token'] = session.access_token + stream_req_data['client']['supportsAdPods'] = stream_type != 'live' + + if stream_type == 'live': + accept_type = 'application/vnd.itv.online.playlist.sim.v3+json' + # Live MUST have a featureset containing an item without outband-webvtt, or a bad request is returned. + min_features = ['mpeg-dash', 'widevine'] + else: + accept_type = 'application/vnd.itv.vod.playlist.v2+json' + # ITV appears now to use the min feature for catchup streams, causing subtitles + # to go missing if not specified here. Min and max both specifying webvtt appears to + # be no problem for catchup streams that don't have subtitles. + min_features = ['mpeg-dash', 'widevine', 'outband-webvtt', 'hd', 'single-track'] + + stream_req_data['variantAvailability']['featureset']['min'] = min_features + + stream_data = fetch_authenticated( + fetch.post_json, url, + data=stream_req_data, + headers={'Accept': accept_type}, + cookies=session.cookie) + + return stream_data def get_live_urls(url=None, title=None, start_time=None, play_from_start=False): @@ -178,12 +161,12 @@ def get_catchup_urls(episode_url): subtitles = stream_data['Subtitles'][0]['Href'] except (TypeError, KeyError, IndexError): subtitles = None - return dash_url, key_service, subtitles, playlist['VideoType'] + return dash_url, key_service, subtitles, playlist['VideoType'], playlist['ProductionId'] def get_vtt_subtitles(subtitles_url): """Return a tuple with the file paths to rst subtitles files. The tuple usually - has only one single element, but could contain more. + has only a single element, but could contain more. Return None if subtitles_url does not point to a valid Web-vvt subtitle file or subtitles are not te be shown by user setting. diff --git a/plugin.video.viwx/resources/lib/itv_account.py b/plugin.video.viwx/resources/lib/itv_account.py index dfbe5cd93..104fec07d 100644 --- a/plugin.video.viwx/resources/lib/itv_account.py +++ b/plugin.video.viwx/resources/lib/itv_account.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later # See LICENSE.txt # ---------------------------------------------------------------------------------------------------------------------- - +import sys import time import os import json @@ -24,6 +24,9 @@ class ItvSession: def __init__(self): + self._user_id = '' + self._user_nickname = '' + self._expire_time = 0 self.account_data = {} self.read_account_data() @@ -34,27 +37,27 @@ def access_token(self): """ try: - if self.account_data['refreshed'] < time.time() - 4 * 3600: - # renew tokens periodically - logger.debug("Token cache time has expired.") - self.refresh() - return self.account_data['itv_session']['access_token'] except (KeyError, TypeError): - logger.debug("Cannot produce access token from account data: %s", self.account_data) - raise AuthenticationError + logger.debug("Cannot produce access token from account data: %s.", self.account_data) + return '' @property def cookie(self): """Return a dict containing the cookie required for authentication""" try: - if self.account_data['refreshed'] < time.time() - 2 * 3600: - # renew tokens periodically - self.refresh() return self.account_data['cookies'] except (KeyError, TypeError): - logger.debug("Cannot produce cookies from account data: %s", self.account_data) - raise AuthenticationError + logger.debug("Cannot produce cookies from account data: %s.", self.account_data) + return {} + + @property + def user_id(self): + return self._user_id or '' + + @property + def user_nickname(self): + return self._user_nickname or '' def read_account_data(self): session_file = os.path.join(utils.addon_info.profile, "itv_session") @@ -74,13 +77,15 @@ def read_account_data(self): self.save_account_data() else: self.account_data = acc_data + access_token = self.account_data.get('itv_session', {}).get('access_token') + self._user_id, self._user_nickname, self._expire_time = parse_token(access_token) def save_account_data(self): session_file = os.path.join(utils.addon_info.profile, "itv_session") data_str = json.dumps(self.account_data) with open(session_file, 'w') as f: f.write(data_str) - logger.info("ITV account data saved to file") + logger.info("ITV account data saved to file.") def login(self, uname: str, passw: str): """Sign in to itv account with `uname` and `passw`. @@ -125,7 +130,8 @@ def login(self, uname: str, passw: str): else: raise else: - logger.info("Sign in successful") + logger.info("Sign in successful.") + self._user_id, self._user_nickname, self._expire_time = parse_token(session_data.get('access_token')) self.save_account_data() return True @@ -153,7 +159,9 @@ def refresh(self): logger.debug("New Itv.Session cookie: %s" % sess_cookie_str) self.account_data['cookies']['Itv.Session'] = sess_cookie_str self.account_data['refreshed'] = time.time() + self._user_id, self._user_nickname, self._expire_time = parse_token(session_data.get('access_token')) self.save_account_data() + logger.info("Tokens refreshed.") return True except (KeyError, ValueError, FetchError) as e: logger.warning("Failed to refresh ITVtokens - %s: %s" % (type(e), e)) @@ -162,11 +170,37 @@ def refresh(self): return False def log_out(self): + logger.info("Signing out to ITV account") self.account_data = {} self.save_account_data() + self._user_id = None + self._user_nickname = None return True +def parse_token(token): + """Return user_id, user nickname and token expiration time obtained from an access token. + + Token has other fields which we currently don't parse, like: + accountProfileIdInUse + auth_time + scope + nonce + iat + """ + import binascii + try: + token_parts = token.split('.') + # Since some padding errors have been observed with refresh tokens, add the maximum just + # to be sure padding errors won't occur. a2b_base64 automatically removes excess padding. + token_data = binascii.a2b_base64(token_parts[1] + '==') + data = json.loads(token_data) + return data['sub'], data['name'], data['exp'] + except (KeyError, AttributeError, IndexError, binascii.Error) as err: + logger.error("Failed to parse token: '%r'", err) + return None, None, int(time.time()) + time.timezone + + def build_cookie(session_data): cookiestr = json.dumps({ 'sticky': True, @@ -185,7 +219,7 @@ def itv_session(): return _itv_session_obj -def fetch_authenticated(funct, url, **kwargs): +def fetch_authenticated(funct, url, login=True, **kwargs): """Call one of the fetch function with user authentication. Call the specified function with authentication header and return the result. @@ -201,21 +235,39 @@ def fetch_authenticated(funct, url, **kwargs): for tries in range(2): try: + access_token = account.access_token + auth_cookies = account.cookie + if not (access_token and auth_cookies): + raise AuthenticationError + + try: + if account.account_data['refreshed'] < time.time() - 4 * 3600: + # renew tokens periodically + logger.debug("Token cache time has expired.") + raise AuthenticationError + except (KeyError, TypeError): + raise AuthenticationError + cookies = kwargs.setdefault('cookies', {}) - cookies.update(account.cookie) + headers = kwargs.setdefault('headers', {}) + headers['authorization'] = 'Bearer ' + account.access_token + cookies.update(auth_cookies) return funct(url=url, **kwargs) except AuthenticationError: - if tries == 0: - logger.debug("Authentication failed on first attempt") - if account.refresh() is False: - logger.debug("") - from . import settings - if not (kodi_utils.show_msg_not_logged_in() and settings.login()): - raise - else: + if tries > 0: logger.warning("Authentication failed on second attempt") raise AccessRestrictedError + logger.debug("Authentication failed on first attempt") + if account.refresh() is False: + if login: + if kodi_utils.show_msg_not_logged_in(): + from xbmc import executebuiltin + executebuiltin('Addon.OpenSettings({})'.format(utils.addon_info.id)) + sys.exit(1) + else: + raise + def convert_session_data(acc_data: dict) -> dict: acc_data['vers'] = SESS_DATA_VERS diff --git a/plugin.video.viwx/resources/lib/itvx.py b/plugin.video.viwx/resources/lib/itvx.py index 2f46c7552..31c1b39aa 100644 --- a/plugin.video.viwx/resources/lib/itvx.py +++ b/plugin.video.viwx/resources/lib/itvx.py @@ -9,16 +9,19 @@ import time import logging -from datetime import datetime import pytz import requests import xbmc +from datetime import datetime, timezone + from codequick.support import logger_id +from . import errors from . import fetch from . import parsex from . import cache +from . import itv_account from .itv import get_live_schedule @@ -38,6 +41,8 @@ def get_page_data(url, cache_time=None): if not url.startswith('https://'): url = 'https://www.itv.com' + url + # URL's with a trailing space have actually happened, but the web app doesn't seem to have a problem with it. + url = url.rstrip() if cache_time: cached_data = cache.get_item(url) if cached_data: @@ -161,80 +166,99 @@ def main_page_items(): def collection_content(url=None, slider=None, hide_paid=False): + """Obtain the collection page defined by `url` and return the contents. If `slider` + is not None, return the contents of that particular slider on the collection page. + + """ uk_tz = pytz.timezone('Europe/London') time_fmt = ' '.join((xbmc.getRegion('dateshort'), xbmc.getRegion('time'))) + is_main_page = url == 'https://www.itv.com' - if url: - # A collection that has its own dedicated page. - page_data = get_page_data(url, cache_time=43200) + page_data = get_page_data(url, cache_time=3600 if is_main_page else 43200) + + if slider: + # Return the contents of the specified slider if slider == 'shortFormSlider': - # return the items from the collection's shortFormSlider + # return the items from the shortFormSlider on a collection page. for item in page_data['shortFormSlider']['items']: yield parsex.parse_shortform_item(item, uk_tz, time_fmt) return - collection = page_data['collection'] - editorial_sliders = page_data.get('editorialSliders') - shortform_slider = page_data.get('shortFormSlider') - - if shortform_slider is not None: - yield parsex.parse_short_form_slider(shortform_slider, url) - - if collection is not None: - for item in collection.get('shows', []): - yield parsex.parse_collection_item(item, hide_paid) - elif editorial_sliders: - # Folders, or kind of sub-collections in a collection. - for slider in editorial_sliders: - yield parsex.parse_slider('', slider) - else: - logger.warning("Missing both collections and editorial_sliders in data from '%s'.", url) - return - - else: - # A Collection that has all it's data on the main page and does not have its own page. - page_data = get_page_data('https://www.itv.com', cache_time=3600) - - if slider == 'shortFormSliderContent': - # Currently only handling News short form. The only other known shorFromSlider is - # 'Sport' and is handled as a full collection. - items_list = None + elif slider == 'shortFormSliderContent': + # Return items form the main page's News short form. The only other known shorFromSlider + # on the main page is 'Sport' and is handled as a full collection. for slider in page_data['shortFormSliderContent']: if slider['key'] == 'newsShortForm': for news_item in slider['items']: yield parsex.parse_shortform_item(news_item, uk_tz, time_fmt, hide_paid) - - if items_list is None: - logger.warning("News shortFormSlider unexpectedly absent from main page") - return + return elif slider == 'trendingSliderContent': + # Only found on main page items_list = page_data['trendingSliderContent']['items'] for trending_item in items_list: yield parsex.parse_trending_collection_item(trending_item, hide_paid) + return else: - try: - items_list = page_data['editorialSliders'][slider]['collection']['shows'] - except KeyError: + # `slider` is the name of an editorialSlider. + # On the main page editorialSliders is a dict, on collection pages it is a list. + # Although a dict on the main page, the names of the sliders are not exactly the + # same as the keys of the dict. + # Until now all editorial sliders on the main page have a 'view all' button, so + # the contents of the slider itself should never be used, but better allow it + # now in case it ever changes. + if is_main_page: + sliders_list = page_data['editorialSliders'].values() + else: + sliders_list = page_data['editorialSliders'] + items_list = None + for slider_item in sliders_list: + if slider_item['collection']['sliderName'] == slider: + items_list = slider_item['collection']['shows'] + break + if items_list is None: logger.error("Failed to parse collection content: Unknown slider '%s'", slider) return for item in items_list: yield parsex.parse_collection_item(item, hide_paid) + else: + # Return the contents of the page, e.i. a listing of individual items for the shortFromSlider + # of the internal collection list, or a list of sub-collections from editorial sliders + collection = page_data['collection'] + editorial_sliders = page_data.get('editorialSliders') + shortform_slider = page_data.get('shortFormSlider') + if shortform_slider is not None: + yield parsex.parse_short_form_slider(shortform_slider, url) + + if collection is not None: + for item in collection.get('shows', []): + yield parsex.parse_collection_item(item, hide_paid) + elif editorial_sliders: + # Folders, or kind of sub-collections in a collection. + for slider in editorial_sliders: + yield parsex.parse_editorial_slider(url, slider) + else: + logger.warning("Missing both collections and editorial_sliders in data from '%s'.", url) + return def episodes(url, use_cache=False): """Get a listing of series and their episodes - Return a list containing only relevant info in a format that can easily be - used by codequick Listitem.from_dict(). + Return a tuple of a series map and a programmeId. + The series map is a dict where keys are series numbers and values are dicts + containing general info regarding the series itself and a list of episodes. + Both formatted in a way that can be used by ListItem.from_dict(). + The programmeId is the programme ID used by ITVX, and is the same for each + series and each episode. """ if use_cache: cached_data = cache.get_item(url) if cached_data is not None: - return cached_data + return cached_data['series_map'], cached_data['programme_id'] page_data = get_page_data(url, cache_time=0) try: @@ -242,6 +266,7 @@ def episodes(url, use_cache=False): except KeyError: logger.warning("Trying to parse episodes in legacy format for programme %s", url) return legacy_episodes(url) + programme_id = programme.get('encodedProgrammeId', {}).get('underscore') programme_title = programme['title'] programme_thumb = programme['image'].format(**parsex.IMG_PROPS_THUMB) programme_fanart = programme['image'].format(**parsex.IMG_PROPS_FANART) @@ -253,10 +278,10 @@ def episodes(url, use_cache=False): series_data = page_data.get('seriesList') if not series_data: - return {} + return {}, None # The field 'seriesNumber' is not guaranteed to be unique - and not guaranteed an integer either. - # Midsummer murder for instance has 2 series with seriesNumber 4 + # Midsummer murder for instance has had 2 series with seriesNumber 4 # By using this mapping, setdefault() and extend() on the episode list, series with the same # seriesNumber are automatically merged. series_map = {} @@ -279,8 +304,10 @@ def episodes(url, use_cache=False): }) series_obj['episodes'].extend( [parsex.parse_episode_title(episode, programme_fanart) for episode in series['titles']]) - cache.set_item(url, series_map, expire_time=1800) - return series_map + + programme_data = {'programme_id': programme_id, 'series_map': series_map} + cache.set_item(url, programme_data, expire_time=1800) + return series_map, programme_id def legacy_episodes(url): @@ -326,20 +353,20 @@ def legacy_episodes(url): }) series_obj['episodes'].extend( [parsex.parse_legacy_episode_title(episode, brand_fanart) for episode in series['episodes']]) - cache.set_item(url, series_map, expire_time=1800) - return series_map + cache.set_item(url, {'programme_id': None, 'series_map': series_map}, expire_time=1800) + return series_map, None def categories(): """Return all available categorie names.""" data = get_page_data('https://www.itv.com/watch/categories', cache_time=86400) cat_list = data['subnav']['items'] - return ({'label': cat['name'], 'params': {'path': cat['url']}} for cat in cat_list) + return ({'label': cat['label'], 'params': {'path': cat['url']}} for cat in cat_list) def category_news(path): """Unlike other categories, news returns a list of sub-categories""" - data = get_page_data(path, cache_time=86400).get('newsData') + data = get_page_data(path, cache_time=86400).get('data') if not data: return [] items = [{'label': 'Latest Stories', 'params': {'path': path, 'subcat': 'heroAndLatestData'}}] @@ -355,7 +382,7 @@ def category_content(url: str, hide_paid=False): if cached_data and cached_data['hide_paid'] == hide_paid: return cached_data['items_list'] - cat_data = get_page_data(url, cache_time=0) + cat_data = get_page_data(url + '/all', cache_time=0) category = cat_data['category']['pathSegment'] progr_list = cat_data.get('programmes') @@ -373,7 +400,7 @@ def category_content(url: str, hide_paid=False): def category_news_content(url, sub_cat, rail=None, hide_paid=False): """Return the content of one of the news sub categories.""" page_data = get_page_data(url, cache_time=900) - news_sub_cats = page_data['newsData'] + news_sub_cats = page_data['data'] uk_tz = pytz.timezone('Europe/London') time_fmt = ' '.join((xbmc.getRegion('dateshort'), xbmc.getRegion('time'))) @@ -431,13 +458,9 @@ def search(search_term, hide_paid=False): """ from urllib.parse import quote - # Include the querystring in url. If requests builds the querystring from parameters it will quote the - # commas in argument `featureset`, and ITV's search appears to have a problem with that and sometimes returns - # no results. url = 'https://textsearch.prd.oasvc.itv.com/search?broadcaster=itv&featureSet=clearkey,outband-webvtt,hls,aes,' \ - 'playready,widevine,fairplay,bbts,progressive,hd,rtmpe&onlyFree=false&platform=dotcom&query=' + quote( - - search_term) + 'playready,widevine,fairplay,bbts,progressive,hd,rtmpe&onlyFree={}&platform=ctv&query={}'.format( + str(hide_paid).lower(), quote(search_term)) headers = { 'User-Agent': fetch.USER_AGENT, 'accept': 'application/json', @@ -466,3 +489,129 @@ def search(search_term, hide_paid=False): if not results: logger.debug("Search for '%s' returned an empty list of results. (hide_paid=%s)", search_term, hide_paid) return (parsex.parse_search_result(result) for result in results) + + +def my_list(user_id, programme_id=None, operation=None, offer_login=True, use_cache=True): + """Get itvX's 'My List', or add or remove an item from 'My List' and return the updated list. + + """ + if operation in ('add', 'remove'): + url = 'https://my-list.prd.user.itv.com/user/{}/mylist/programme/{}?features={}&platform={}'.format( + user_id, programme_id, FEATURE_SET, PLATFORM_TAG) + else: + cached_list = cache.get_item('mylist_' + user_id) + if use_cache and cached_list is not None: + return cached_list + else: + url = 'https://my-list.prd.user.itv.com/user/{}/mylist?features={}&platform={}'.format( + user_id, FEATURE_SET, PLATFORM_TAG) + + fetcher = { + 'get': fetch.get_json, + 'add': fetch.post_json, + 'remove': fetch.delete_json}.get(operation, fetch.get_json) + + data = itv_account.fetch_authenticated(fetcher, url, data=None, login=offer_login) + # Empty lists will return HTTP status 204, which results in data being None. + if data: + my_list_items = [parsex.parse_my_list_item(item) for item in data] + else: + my_list_items = [] + cache.set_item('mylist_' + user_id, my_list_items, 1800) + cache.my_list_programmes = list(item['programme_id'] for item in my_list_items) + return my_list_items + + +def initialise_my_list(): + """Get all items from itvX's 'My List'. + Used when the module is first imported, or after account sign in to initialise + the cached list of programme ID's before the plugin lists programmes. + + """ + try: + my_list(itv_account.itv_session().user_id, offer_login=False, use_cache=False) + logger.info("Updated MyList programme ID's.") + except Exception as err: + # Since this runs before codequick.run() all exceptions must be caught to prevent them + # crashing the addon before the main menu is shown. + # Most likely the user is not (yet) logged in, but at the start of the addon connection + # errors could also occur, depending on the network setup. + logger.info("Failed to update MyList programme ID's: %r.", err) + pass + + +def get_last_watched(): + user_id = itv_account.itv_session().user_id + cache_key = 'last_watched_' + user_id + cached_data = cache.get_item(cache_key) + if cached_data is not None: + return cached_data + + url = 'https://content.prd.user.itv.com/lastwatched/user/{}/{}?features={}'.format( + user_id, PLATFORM_TAG, FEATURE_SET) + header = {'accept': 'application/vnd.user.content.v1+json'} + utc_now = datetime.now(tz=timezone.utc).replace(tzinfo=None) + data = itv_account.fetch_authenticated(fetch.get_json, url, headers=header) + watched_list = [parsex.parse_last_watched_item(item, utc_now) for item in data] + cache.set_item(cache_key, watched_list, 600) + return watched_list + + +def get_resume_point(production_id: str): + try: + production_id = production_id.replace('/', '_').replace('#', '.') + url = 'https://content.prd.user.itv.com/resume/user/{}/productionid/{}'.format( + itv_account.itv_session().user_id, production_id) + data = itv_account.fetch_authenticated(fetch.get_json, url) + resume_time = data['progress']['time'].split(':') + resume_point = int(resume_time[0]) * 3600 + int(resume_time[1]) * 60 + float(resume_time[2]) + return resume_point + except errors.HttpError as err: + if err.code == 404: + # Normal response when no resume data is found, e.g. with 'next episodes'. + logger.debug("Resume point of production '%s' not available.", production_id) + else: + logger.error("HTTP error %s: %s on request for resume point of production '%s'.", + err.code, err.reason, production_id) + except: + logger.error("Error obtaining resume point of production '%s'.", production_id, exc_info=True) + return None + + +def recommended(user_id, hide_paid=False): + """Get the list of recommendations from ITVX. + Always returns data, even if user_id is invalid or absent. + + """ + recommended_url = 'https://recommendations.prd.user.itv.com/recommendations/homepage/' + user_id + + recommended = cache.get_item(recommended_url) + if not recommended: + req_params = {'features': FEATURE_SET, 'platform': PLATFORM_TAG, 'size': 24} + recommended = fetch.get_json(recommended_url, params=req_params) + if not recommended: + return None + cache.set_item(recommended_url, recommended, 43200) + return list(filter(None, (parsex.parse_my_list_item(item, hide_paid) for item in recommended))) + + +def because_you_watched(user_id, name_only=False, hide_paid=False): + """Return the list of recommendation based on the last watched programme. + + Returns 204 - No Content when user ID is invalid. Doesn't require authentication. + """ + if not user_id: + return + byw_url = 'https://recommendations.prd.user.itv.com/recommendations/byw/' + user_id + byw = cache.get_item(byw_url) + if not byw: + req_params = {'features': FEATURE_SET, 'platform': PLATFORM_TAG, 'size': 12} + byw = fetch.get_json(byw_url, params=req_params) + if not byw: + return + cache.set_item(byw_url, byw, 1800) + + if name_only: + return byw['watched_programme'] + else: + return list(filter(None, (parsex.parse_my_list_item(item, hide_paid) for item in byw['recommendations']))) diff --git a/plugin.video.viwx/resources/lib/kodi_utils.py b/plugin.video.viwx/resources/lib/kodi_utils.py index 5e9f9e997..3c87769d1 100644 --- a/plugin.video.viwx/resources/lib/kodi_utils.py +++ b/plugin.video.viwx/resources/lib/kodi_utils.py @@ -70,6 +70,7 @@ def show_msg_not_logged_in(): Script.localize(MSG_LOGIN), nolabel=Script.localize(BTN_TXT_CANCEL), yeslabel=Script.localize(TXT_LOGIN_NOW)) + logger.debug("Dialog 'Open settings to login' result: {}".format('YES' if result else 'NO' )) return result @@ -85,7 +86,7 @@ def show_login_result(success: bool, message: str = None): def ask_login_retry(reason): - """Show a message that login has failed and ask whether to try again""" + """Show a message that login has failed and ask whether to try again.""" if reason.lower() == 'invalid username': reason = Script.localize(TXT_INVALID_USERNAME) diff --git a/plugin.video.viwx/resources/lib/main.py b/plugin.video.viwx/resources/lib/main.py index a317cc784..092556005 100644 --- a/plugin.video.viwx/resources/lib/main.py +++ b/plugin.video.viwx/resources/lib/main.py @@ -12,18 +12,20 @@ import pytz import requests +import xbmc import xbmcplugin from xbmcgui import ListItem from codequick import Route, Resolver, Listitem, Script, run as cc_run from codequick.support import logger_id, build_path, dispatcher -from resources.lib import itv, itvx +from resources.lib import itv, itv_account, itvx from resources.lib import utils from resources.lib import parsex from resources.lib import fetch from resources.lib import kodi_utils from resources.lib import cache +from resources.lib import xprogress from resources.lib.errors import * @@ -91,9 +93,17 @@ def __init__(self, items_list, filter_char, page_nr, **kwargs): self._filter = filter_char self._page_nr = page_nr self._kwargs = kwargs + self._is_az_list = None self._addon = utils.addon_info.addon - def _show_az_list(self): + @property + def is_az_list(self): + """True if the paginator will return only A to Z folders""" + if self._is_az_list is None: + self._is_az_list = self._get_show_az_list() + return self._is_az_list + + def _get_show_az_list(self): az_len = self._addon.getSettingInt('a-z_size') items_list = self._items_list try: @@ -147,7 +157,11 @@ def _generate_page(self): for show in shows_list: try: - yield Listitem.from_dict(callb_map[show['type']], **show['show']) + li = Listitem.from_dict(callb_map[show['type']], **show['show']) + # Create 'My List' add/remove context menu entries here, so as to be able to update these + # entries after adding/removing an item, even when the underlying data is cached. + _my_list_context_mnu(li, show.get('programme_id')) + yield li except KeyError: logger.warning("Cannot list '%s': unknown item type '%s'", show['show'].get('info', {}).get('sorttitle', ''), show['type']) @@ -158,7 +172,7 @@ def _generate_page(self): yield li def __iter__(self): - if self._show_az_list(): + if self.is_az_list: return self._generate_az() else: return self._generate_page() @@ -166,15 +180,74 @@ def __iter__(self): @Route.register(content_type='videos') def root(_): + yield Listitem.from_dict(sub_menu_my_itvx, 'My itvX') yield Listitem.from_dict(sub_menu_live, 'Live', params={'_cache_to_disc_': False}) for item in itvx.main_page_items(): callback = callb_map.get(item['type'], play_title) - yield Listitem.from_dict(callback, **item['show']) + li = Listitem.from_dict(callback, **item['show']) + _my_list_context_mnu(li, item.get('programme_id')) + yield li yield Listitem.from_dict(list_collections, 'Collections') yield Listitem.from_dict(list_categories, 'Categories') yield Listitem.search(do_search, Script.localize(TXT_SEARCH)) +@Route.register(content_type='videos') +def sub_menu_my_itvx(_): + # Ensure to add at least one parameter to persuade dynamic listing that we actually call the list. + yield Listitem.from_dict(generic_list, 'My List', params={'list_type':'mylist', 'filter_char': None}) + yield Listitem.from_dict(generic_list, 'Continue Watching', params={'list_type':'watching', 'filter_char': None}) + last_programme = itvx.because_you_watched(itv_account.itv_session().user_id, name_only=True) + if last_programme: + yield Listitem.from_dict(generic_list, 'Because You Watched ' + last_programme, params={'list_type':'byw'}) + yield Listitem.from_dict(generic_list, 'Recommended for You', params={'list_type':'recommended'}) + + +def _my_list_context_mnu(list_item, programme_id, refresh=True): + """If `list_item` contains a programme_id, check if the id is in 'My List' + and add a context menu to add or remove the item from the list accordingly. + + """ + if not programme_id: + return + try: + if programme_id in cache.my_list_programmes: + list_item.context.script(update_mylist, "Remove from My List", + progr_id=programme_id, operation='remove', refresh=refresh) + else: + list_item.context.script(update_mylist, "Add to My List", + progr_id=programme_id, operation='add', refresh=refresh) + except TypeError: + # The cached list of programme ID's is not intialised; do not set a context menu + logger.warning("Cannot create 'My List' context menu") + + +@Route.register(content_type='videos') +@dynamic_listing +def generic_list(addon, list_type='mylist', filter_char=None, page_nr=0): + """List the contents of itvX's 'My List', 'Continue Watching', 'Because You Watched' and 'Recommended'. + + """ + addon.add_sort_methods(xbmcplugin.SORT_METHOD_UNSORTED, + xbmcplugin.SORT_METHOD_TITLE, + disable_autosort=True) + if list_type == 'mylist': + addon.add_sort_methods(xbmcplugin.SORT_METHOD_DATE) + shows_list = itvx.my_list(itv_account.itv_session().user_id) + elif list_type == 'watching': + addon.add_sort_methods(xbmcplugin.SORT_METHOD_DATE) + shows_list = itvx.get_last_watched() + elif list_type == 'byw': + shows_list = itvx.because_you_watched(itv_account.itv_session().user_id, + hide_paid=addon.setting.get_boolean('hide_paid')) + elif list_type == 'recommended': + shows_list = itvx.recommended(itv_account.itv_session().user_id, + hide_paid=addon.setting.get_boolean('hide_paid')) + else: + raise ValueError(f"Unknown generic list type: '{list_type}'.") + yield from Paginator(shows_list, filter_char, page_nr) + + @Route.register(content_type='videos') def sub_menu_live(_): try: @@ -237,7 +310,9 @@ def sub_menu_live(_): @Route.register(content_type='videos') def list_collections(_): - main_page = itvx.get_page_data('https://www.itv.com', cache_time=3600) + """A list of all available collections.""" + url ='https://www.itv.com' + main_page = itvx.get_page_data(url, cache_time=3600) for slider in main_page['shortFormSliderContent']: if slider['key'] == 'newsShortForm': # News is already on the home page by default. @@ -246,8 +321,8 @@ def list_collections(_): if item: yield Listitem.from_dict(list_collection_content, **item['show']) - for slider in main_page['editorialSliders'].items(): - item = parsex.parse_slider(*slider) + for slider in main_page['editorialSliders'].values(): + item = parsex.parse_editorial_slider(url, slider) if item: yield Listitem.from_dict(list_collection_content, **item['show']) @@ -327,10 +402,12 @@ def list_productions(plugin, url, series_idx=None): xbmcplugin.SORT_METHOD_DATE, disable_autosort=True) - series_map = itvx.episodes(url, use_cache=True) - if not series_map: + result = itvx.episodes(url, use_cache=True) + if not result: return + series_map, programme_id = result + if len(series_map) == 1: # List the episodes if there is only 1 series opened_series = list(series_map.values())[0] @@ -353,6 +430,7 @@ def list_productions(plugin, url, series_idx=None): # List folders of all series for series in series_map.values(): li = Listitem.from_dict(list_productions, **series['series']) + _my_list_context_mnu(li, programme_id) yield li @@ -363,11 +441,12 @@ def do_search(addon, search_query): if not search_results: return - items = [ - Listitem.from_dict(callb_map.get(result['type'], play_title), **result['show']) - for result in search_results if result is not None - ] - return items + for result in search_results: + if result is None: + continue + li = Listitem.from_dict(callb_map.get(result['type'], play_title), **result['show']) + _my_list_context_mnu(li, result['programme_id'], refresh=False) + yield li def create_dash_stream_item(name: str, manifest_url, key_service_url, resume_time=None): @@ -377,20 +456,15 @@ def create_dash_stream_item(name: str, manifest_url, key_service_url, resume_tim logger.debug('dash manifest url: %s', manifest_url) logger.debug('dash key service url: %s', key_service_url) - try: - # Ensure to get a fresh hdntl cookie as they expire after 12 or 24 hrs. - # Use a plain requests.get() to prevent sending an existing hdntl cookie, - # and other cookies are not required. - resp = requests.get(url=manifest_url, - allow_redirects=False, - headers={'user-agent': fetch.USER_AGENT}, - timeout=fetch.WEB_TIMEOUT) - hdntl_cookie = resp.cookies.get('hdntl', '') - logger.debug("Received hdntl cookie: %s", hdntl_cookie) - except FetchError as err: - logger.error('Error retrieving dash manifest - url: %r' % err) - Script.notify('ITV', str(err), Script.NOTIFY_ERROR) - return False + # Ensure to get a fresh hdntl cookie as they expire after 12 or 24 hrs. + # Use a plain requests.get() to prevent sending an existing hdntl cookie, + # and other cookies are not required. + resp = requests.get(url=manifest_url, + allow_redirects=False, + headers={'user-agent': fetch.USER_AGENT}, + timeout=fetch.WEB_TIMEOUT) + hdntl_cookie = resp.cookies.get('hdntl', '') + logger.debug("Received hdntl cookie: %s", hdntl_cookie) PROTOCOL = 'mpd' DRM = 'com.widevine.alpha' @@ -460,23 +534,14 @@ def play_stream_live(addon, channel, url, title=None, start_time=None, play_from if addon.setting['live_play_from_start'] != 'true' and not play_from_start: start_time = None - try: - manifest_url, key_service_url, subtitle_url = itv.get_live_urls(url, - title, - start_time, - play_from_start) - except FetchError as err: - logger.error('Error retrieving live stream urls: %r' % err) - Script.notify('ITV', str(err), Script.NOTIFY_ERROR) - return False - except Exception as e: - logger.error('Error retrieving live stream urls: %r' % e, exc_info=True) - return - + manifest_url, key_service_url, subtitle_url = itv.get_live_urls(url, + title, + start_time, + play_from_start) list_item = create_dash_stream_item(channel, manifest_url, key_service_url) if list_item: # list_item.setProperty('inputstream.adaptive.manifest_update_parameter', 'full') - if 't=' in manifest_url: + if '?t=' in manifest_url or '&t=' in manifest_url: list_item.setProperty('inputstream.adaptive.play_timeshift_buffer', 'true') # list_item.property['inputstream.adaptive.live_delay'] = '2' logger.debug("play live stream - timeshift_buffer enabled") @@ -486,35 +551,40 @@ def play_stream_live(addon, channel, url, title=None, start_time=None, play_from @Resolver.register -def play_stream_catchup(_, url, name): +def play_stream_catchup(plugin, url, name, set_resume_point=False): logger.info('play catchup stream - %s url=%s', name, url) try: - manifest_url, key_service_url, subtitle_url, stream_type = itv.get_catchup_urls(url) + manifest_url, key_service_url, subtitle_url, stream_type, production_id = itv.get_catchup_urls(url) logger.debug('dash subtitles url: %s', subtitle_url) except AccessRestrictedError: logger.info('Stream only available with premium account') kodi_utils.msg_dlg(Script.localize(TXT_PREMIUM_CONTENT)) return False - except FetchError as err: - logger.error('Error retrieving episode stream urls: %r' % err) - Script.notify(utils.addon_info.name, str(err), Script.NOTIFY_ERROR) - return False - except Exception: - logger.error('Error retrieving catchup stream urls:', exc_info=True) - return False if stream_type == 'SHORT': return create_mp4_file_item(name, manifest_url) else: list_item = create_dash_stream_item(name, manifest_url, key_service_url) + if not list_item: + return False + + plugin.register_delayed(xprogress.playtime_monitor, production_id=production_id) subtitles = itv.get_vtt_subtitles(subtitle_url) - if list_item and subtitles: + if subtitles: list_item.setSubtitles(subtitles) list_item.setProperties({ 'subtitles.translate.file': subtitles[0], 'subtitles.translate.orig_lang': 'en', 'subtitles.translate.type': 'srt'}) + if set_resume_point: + resume_time = itvx.get_resume_point(production_id) + if resume_time: + list_item.setProperties({ + 'ResumeTime': str(resume_time), + 'TotalTime': '7200' + }) + logger.info("Resume from %s", resume_time) return list_item @@ -535,6 +605,28 @@ def play_title(plugin, url, name=''): return play_stream_catchup(plugin, url, name) +@Script.register +def update_mylist(_, progr_id, operation, refresh=True): + """Context menu handler to add or remove a programme from itvX's 'My List'. + + @param str progr_id: The underscore encoded programme ID. + @param str operation: The operation to apply; either 'add' or 'remove'. + @param bool refresh: whether to perform a Container.Refresh + + """ + try: + itvx.my_list(itv_account.itv_session().user_id, progr_id, operation) + except (ValueError, IndexError, FetchError): + if operation == 'add': + kodi_utils.msg_dlg('Failed to add this item to My List', 'My List Error') + else: + kodi_utils.msg_dlg('Failed to remove this item from My List', 'My List Error') + return + logger.info("Updated MyList: %s programme %s", operation, progr_id) + if refresh: + xbmc.executebuiltin('Container.Refresh') + + def run(): if isinstance(cc_run(), Exception): xbmcplugin.endOfDirectory(int(sys.argv[1]), False) @@ -555,5 +647,12 @@ def run(): 'episode': play_title, 'special': play_title, 'film': play_title, - 'title': play_title + 'title': play_title, + 'vodstream': play_stream_catchup } + + +# A rather hacky method to ensure the cached list of programmeId's in itvx's My List +# is updated each time the addon starts, but not on settings callbacks. +if cache.my_list_programmes is None and 'resources/lib/settings' not in sys.argv[0]: + itvx.initialise_my_list() diff --git a/plugin.video.viwx/resources/lib/parsex.py b/plugin.video.viwx/resources/lib/parsex.py index d66e51073..87cceda5c 100644 --- a/plugin.video.viwx/resources/lib/parsex.py +++ b/plugin.video.viwx/resources/lib/parsex.py @@ -9,6 +9,7 @@ import json import logging import pytz +from datetime import datetime from codequick.support import logger_id @@ -82,6 +83,13 @@ def parse_hero_content(hero_data): try: item_type = hero_data['contentType'] title = hero_data['title'] + + if item_type in ('collection', 'page'): + item = parse_item_type_collection(hero_data) + info = item['show']['info'] + info['title'] = ''.join(('[COLOR orange]', info['title'], '[/COLOR]')) + return item + item = { 'label': title, 'art': {'thumb': hero_data['imageTemplate'].format(**IMG_PROPS_THUMB), @@ -111,15 +119,12 @@ def parse_hero_content(hero_data): item['params'] = {'url': build_url(title, hero_data['encodedProgrammeId']['letterA']), 'name': title} - elif item_type == 'collection': - item = parse_item_type_collection(hero_data) - info = item['show']['info'] - info['title'] = ''.join(('[COLOR orange]', info['title'], '[/COLOR]')) - return item else: logger.warning("Hero item %s is of unknown type: %s", hero_data['title'], item_type) return None - return {'type': item_type, 'show': item} + return {'type': item_type, + 'programme_id': hero_data.get('encodedProgrammeId', {}).get('underscore'), + 'show': item} except: logger.warning("Failed to parse hero item '%s':\n", hero_data.get('title', 'unknown title'), exc_info=True) @@ -155,26 +160,31 @@ def parse_short_form_slider(slider_data, url=None): return None -def parse_slider(slider_name, slider_data): +def parse_editorial_slider(url, slider_data): """Parse editorialSliders from the main page or from a collection.""" # noinspection PyBroadException try: coll_data = slider_data['collection'] + if not coll_data.get('shows'): + # Has happened. Items without field `shows` have an invalid headingLink + return page_link = coll_data.get('headingLink') base_url = 'https://www.itv.com/watch' if page_link: # Link to the collection's page if available params = {'url': base_url + page_link['href']} else: - # Provide the slider name when the collection content is to be obtained from the main page. - params = {'slider': slider_name} + # Provide the slider name when the collection contents are the + # items in the slider on the original page. + slider_name = slider_data['collection']['sliderName'] + params = {'url': url, 'slider': slider_name} return {'type': 'collection', 'show': {'label': coll_data['headingTitle'], 'params': params, 'info': {'sorttitle': sort_title(coll_data['headingTitle'])}}} except: - logger.error("Unexpected error parsing editorialSlider %s", slider_name, exc_info=True) + logger.error("Unexpected error parsing editorialSlider from %s", url, exc_info=True) return None @@ -191,7 +201,7 @@ def parse_collection_item(show_data, hide_paid=False): title = show_data['title'] content_info = show_data.get('contentInfo', '') - if content_type == 'collection': + if content_type in ('collection', 'page'): return parse_item_type_collection(show_data) if show_data.get('isPaid'): @@ -224,6 +234,7 @@ def parse_collection_item(show_data, hide_paid=False): if is_playable: programme_item['info']['duration'] = utils.duration_2_seconds(content_info) return {'type': content_type, + 'programme_id': show_data.get('encodedProgrammeId', {}).get('underscore'), 'show': programme_item} except Exception as err: logger.warning("Failed to parse collection_item: %r\n%s", err, json.dumps(show_data, indent=4)) @@ -237,9 +248,6 @@ def parse_shortform_item(item_data, time_zone, time_fmt, hide_paid=False): ShortFormSliders are found on the main page, some collection pages. Items from heroAndLatest and curatedRails in category news also have a shortForm-like content. - """ - """Parse data found in news collection and in short news clips from news sub-categories - """ try: if 'encodedProgrammeId' in item_data.keys(): @@ -304,6 +312,7 @@ def parse_trending_collection_item(trending_item, hide_paid=False): return { 'type': 'title', + 'programme_id': trending_item['encodedProgrammeId']['underscore'], 'show': { 'label': trending_item['title'], 'art': {'thumb': trending_item['imageUrl'].format(**IMG_PROPS_THUMB)}, @@ -329,6 +338,8 @@ def parse_category_item(prog, category): # All items with episodeId are returned as series folder, with the odd change some # contain only one item. + # TODO: Both regular and news category items now have a field contentType + is_playable = prog['encodedEpisodeId']['letterA'] == '' playtime = utils.duration_2_seconds(prog['contentInfo']) title = prog['title'] @@ -359,17 +370,26 @@ def parse_category_item(prog, category): prog['encodedProgrammeId']['letterA'], prog['encodedEpisodeId']['letterA'])} return {'type': 'title' if is_playable else 'series', + 'programme_id': prog['encodedProgrammeId']['underscore'], 'show': programme_item} def parse_item_type_collection(item_data): - """Parse an item of type 'collection' found in heroContent or a collection. + """Parse an item of type 'collection' or type 'page' found in heroContent or + a collection. The collection items refer to another collection. .. note:: Only items from heroContent seem to have a field `ctaLabel`. """ + url = '/'.join(('https://www.itv.com/watch/collections', + item_data.get('titleSlug', ''), + item_data.get('collectionId') or item_data['pageId'])) + if item_data['contentType'] == 'page': + # This querystring is required for page items + url += '?ind' + title = item_data['title'] item = { 'label': title, @@ -378,9 +398,7 @@ def parse_item_type_collection(item_data): 'info': {'title': '[B]{}[/B]'.format(title), 'plot': item_data.get('ctaLabel', 'Collection'), 'sorttitle': sort_title(title)}, - 'params': {'url': '/'.join(('https://www.itv.com/watch/collections', - item_data.get('titleSlug', ''), - item_data.get('collectionId')))} + 'params': {'url': url} } return {'type': 'collection', 'show': item} @@ -400,6 +418,10 @@ def parse_episode_title(title_data, brand_fanart=None): else: info_title = title_data['heroCtaLabel'] + series_nr = title_data.get('series') + if not isinstance(series_nr, int): + series_nr = None + title_obj = { 'label': title, 'art': {'thumb': img_url.format(**IMG_PROPS_THUMB), @@ -411,7 +433,7 @@ def parse_episode_title(title_data, brand_fanart=None): 'duration': utils.iso_duration_2_seconds(title_data['notFormattedDuration']), 'date': title_data['dateTime'], 'episode': episode_nr, - 'season': title_data.get('series'), + 'season': series_nr, 'year': title_data.get('productionYear')}, 'params': {'url': title_data['playlistUrl'], 'name': title} } @@ -466,7 +488,7 @@ def parse_search_result(search_data): prog_name = result_data['programmeTitle'] title = '[B]{}[/B] - {} episodes'.format(prog_name, result_data.get('totalAvailableEpisodes', '')) img_url = result_data['latestAvailableEpisode']['imageHref'] - api_prod_id = result_data['legacyId']['officialFormat'] + api_prod_id = result_data['legacyId']['apiEncoded'] elif entity_type == 'special': # A single programme without episodes @@ -475,18 +497,22 @@ def parse_search_result(search_data): programme = result_data.get('specialProgramme') if programme: - prog_name = result_data['specialProgramme']['programmeTitle'] - api_prod_id = result_data['specialProgramme']['legacyId']['officialFormat'] + prog_name = programme['programmeTitle'] + api_prod_id = programme['legacyId']['apiEncoded'] api_episode_id = result_data['legacyId']['officialFormat'] else: prog_name = title - api_prod_id = result_data['legacyId']['officialFormat'] + api_prod_id = result_data['legacyId']['apiEncoded'] + if api_prod_id.count('_') > 1: + api_prod_id = api_prod_id.rpartition('_')[0] elif entity_type == 'film': prog_name = result_data['filmTitle'] title = '[B]Film[/B] - ' + result_data['filmTitle'] img_url = result_data['imageHref'] - api_prod_id = result_data['legacyId']['officialFormat'] + api_prod_id = result_data['legacyId']['apiEncoded'] + if api_prod_id.count('_') > 1: + api_prod_id = api_prod_id.rpartition('_')[0] else: logger.warning("Unknown search result item entityType %s", entity_type) @@ -494,11 +520,119 @@ def parse_search_result(search_data): return { 'type': entity_type, + 'programme_id': api_prod_id, 'show': { 'label': prog_name, 'art': {'thumb': img_url.format(**IMG_PROPS_THUMB)}, 'info': {'plot': plot, 'title': title}, - 'params': {'url': build_url(prog_name, api_prod_id.replace('/', 'a'), api_episode_id.replace('/', 'a'))} + 'params': {'url': build_url(prog_name, api_prod_id.replace('_', 'a'), api_episode_id.replace('/', 'a'))} + } + } + + +def parse_my_list_item(item, hide_paid=False): + """Parser for items from My List, Recommended and Because You Watched.""" + # noinspection PyBroadException + try: + if 'PAID' in item['tier']: + if hide_paid: + return None + description = premium_plot(item['synopsis']) + else: + description = item['synopsis'] + progr_name = item.get('programmeTitle') or item['title'] + progr_id = item['programmeId'].replace('/', '_') + num_episodes = item['numberOfEpisodes'] + content_info = ' - {} episodes'.format(num_episodes) if num_episodes is not None else '' + img_link = item.get('itvxImageLink') or item.get('imageUrl') + is_playable = item['contentType'].lower() != 'programme' + + item_dict = { + 'type': item['contentType'].lower(), + 'programme_id': progr_id, + 'show': { + 'label': progr_name, + 'art': {'thumb': img_link.format(**IMG_PROPS_THUMB), + 'fanart': img_link.format(**IMG_PROPS_FANART)}, + 'info': {'title': progr_name if is_playable else '[B]{}[/B]{}'.format(progr_name, content_info), + 'plot': description, + 'duration': utils.iso_duration_2_seconds(item.get('duration')), + 'sorttitle': sort_title(progr_name), + 'date': item.get('dateAdded')}, + 'params': {'url': build_url(progr_name, progr_id.replace('/', 'a'))} + } + } + if item['contentType'] == 'FILM': + item_dict['show']['art']['poster'] = img_link.format(**IMG_PROPS_POSTER) + return item_dict + except: + logger.warning("Unexpected error parsing MyList item:\n", exc_info=True) + + +def parse_last_watched_item(item, utc_now): + progr_name = item.get('programmeTitle', '') + episode_name = item.get('episodeTitle') + series_nr = item.get('seriesNumber') + episode_nr = item.get('episodeNumber') + img_link = item.get('itvxImageLink', '') + available_td = utils.strptime(item['availabilityEnd'], "%Y-%m-%dT%H:%M:%SZ") - utc_now + days_available = int(available_td.days + 0.99) + + if days_available > 365: + availability = '\nAvailable for over a year.' + elif days_available > 30: + months = int(days_available//30) + availability = '\nAvailable for {} month{}.'.format(months, 's' if months > 1 else '') + elif days_available >= 1: + availability = '\n[COLOR orange]Only {} day{} available.[/COLOR]'.format( + days_available, 's' if days_available > 1 else '') + else: + hours_available = int(available_td.seconds / 3600) + availability = '\n[COLOR orange]Only {} hour{} available.[/COLOR]'.format( + hours_available, 's' if hours_available != 1 else '') + + info = ''.join(( + item['synopsis'] if 'FREE' in item['tier'] else premium_plot(item['synopsis']), + '\n\n', + episode_name or '', + ' - ' if episode_name and series_nr else '', + 'series {} episode {}'.format(series_nr, episode_nr) if series_nr else '', + availability + )) + + if item.get('isNextEpisode'): + title = progr_name + ' - [I]next episode[/I]' + else: + title = '{} - [I]{}% watched[/I]'.format(progr_name, int(item['percentageWatched'] * 100)) + + item_dict = { + 'type': 'vodstream', + 'programme_id': item['programmeId'].replace('/', '_'), + 'show': { + 'label': episode_name or progr_name, + 'art': {'thumb': img_link.format(**IMG_PROPS_THUMB), + 'fanart': img_link.format(**IMG_PROPS_FANART)}, + 'info': {'title': title , + 'plot': info, + 'sorttitle': sort_title(title), + 'date': utils.reformat_date(item['viewedOn'], "%Y-%m-%dT%H:%M:%SZ", "%d.%m.%Y"), + 'duration': utils.duration_2_seconds(item['duration']), + 'season': series_nr, + 'episode': episode_nr}, + 'params': {'url': ('https://magni.itv.com/playlist/itvonline/ITV/' + + item['productionId'].replace('/', '_').replace('#', '.' )), + 'name': progr_name, + 'set_resume_point': True}, + 'properties': { + # This causes Kodi not to offer the standard resume dialog, so we can obtain + # resume time at the time of resolving the video url and play from there, or show + # a 'resume from' dialog. + 'resumetime': '0', + 'totaltime': 60 + } } } + if item['contentType'] == 'FILM': + item_dict['show']['art']['poster'] = img_link.format(**IMG_PROPS_POSTER) + return item_dict \ No newline at end of file diff --git a/plugin.video.viwx/resources/lib/settings.py b/plugin.video.viwx/resources/lib/settings.py index 4bded7f12..4de410989 100644 --- a/plugin.video.viwx/resources/lib/settings.py +++ b/plugin.video.viwx/resources/lib/settings.py @@ -29,6 +29,7 @@ def login(_=None): uname = None passw = None + logger.debug("Starting Login...") while True: uname, passw = kodi_utils.ask_credentials(uname, passw) if not all((uname, passw)): @@ -37,6 +38,10 @@ def login(_=None): try: itv_account.itv_session().login(uname, passw) kodi_utils.show_login_result(success=True) + from resources.lib import itvx + import xbmc + itvx.initialise_my_list() + xbmc.executebuiltin('Container.Refresh') return except errors.AuthenticationError as e: if not kodi_utils.ask_login_retry(str(e)): @@ -51,6 +56,10 @@ def logout(_): Script.notify(Script.localize(kodi_utils.TXT_ITV_ACCOUNT), Script.localize(kodi_utils.MSG_LOGGED_OUT_SUCCESS), Script.NOTIFY_INFO) + from resources.lib import cache + import xbmc + cache.my_list_programmes = None + xbmc.executebuiltin('Container.Refresh') @Script.register() diff --git a/plugin.video.viwx/resources/lib/xprogress.py b/plugin.video.viwx/resources/lib/xprogress.py new file mode 100644 index 000000000..14d7af851 --- /dev/null +++ b/plugin.video.viwx/resources/lib/xprogress.py @@ -0,0 +1,253 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Copyright (c) 2023 Dimitri Kroon. +# This file is part of plugin.video.viwx. +# SPDX-License-Identifier: GPL-2.0-or-later +# See LICENSE.txt +# ---------------------------------------------------------------------------------------------------------------------- +import threading +import uuid +import time +import logging + +from xbmc import Player, Monitor + +from codequick.support import logger_id + +from resources.lib import fetch +from . itv_account import itv_session +from . itvx import PLATFORM_TAG + + +logger = logging.getLogger('.'.join((logger_id, __name__.split('.', 2)[-1]))) + + +EVT_URL = 'https://secure.pes.itv.com/1.1.3/event' + + +class PlayState: + UNDEFINED = 0xFF00 + PLAYING = 0xFF01 + PAUSED = 0xFF02 + STOPPED = 0xFF03 + + +class PlayTimeMonitor(Player): + POLL_PERIOD = 1 + REPORT_PERIOD = 30 + + def __init__(self, production_id): + super(PlayTimeMonitor, self).__init__() + self._instance_id = None + self._production_id = production_id + self._event_seq_nr = 0 + self._playtime = 0 + self._user_id = itv_session().user_id + self.monitor = Monitor() + self._status = PlayState.UNDEFINED + self._cur_file = None + self._post_errors = 0 + + @property + def playtime(self): + """Return the last known playtime in milliseconds""" + return int(self._playtime * 1000) + + def onAVStarted(self) -> None: + # noinspection PyBroadException + logger.debug("onAVStarted called from thread %s", threading.current_thread().native_id) + if self._status is not PlayState.UNDEFINED: + logger.warning("onAvStarted - player is already initialised") + return + + try: + self._cur_file = self.getPlayingFile() + self._playtime = self.getTime() + self._status = PlayState.PLAYING + logger.debug("PlayTimeMonitor: total play time = %s", self.playtime/60) + self._post_event_startup_complete() + except: + logger.error("PlayTimeMonitor.onAVStarted:\n", exc_info=True) + self._playtime = 0 + self._status = PlayState.STOPPED + + def onAVChange(self) -> None: + if self._cur_file and self._cur_file != self.getPlayingFile(): + logger.debug("onAvChange: playing has stopped. Now playing file '%s'", self.getPlayingFile()) + self.onPlayBackStopped() + + def onPlayBackStopped(self) -> None: + cur_state = self._status + self._status = PlayState.STOPPED + if cur_state in (PlayState.UNDEFINED, PlayState.STOPPED): + return + self._post_event_heartbeat() + + def onPlayBackEnded(self) -> None: + self.onPlayBackStopped() + + def onPlayBackError(self) -> None: + self.onPlayBackStopped() + + def wait_until_playing(self, timeout) -> bool: + """Wait and return `True` when the player has started playing. + Return `False` when `timeout` expires, or when playing has been aborted before + the actual playing started. + + """ + end_t = time.monotonic() + timeout + while self._status is PlayState.UNDEFINED: + if time.monotonic() >= end_t: + return False + if self.monitor.waitForAbort(0.2): + logger.debug("wait_until_playing ended: abort requested") + return False + return not self._status is PlayState.STOPPED + + def monitor_progress(self) -> None: + """Wait while the player is playing and return when playing the file has stopped. + Returns immediately if the player is not playing. + + """ + if self._status is PlayState.UNDEFINED: + return + logger.debug("Playtime Monitor start") + report_t = time.monotonic() + self.REPORT_PERIOD + while not (self.monitor.waitForAbort(self.POLL_PERIOD) or self._status is PlayState.STOPPED): + try: + self._playtime = self.getTime() + except RuntimeError: # Player just stopped playing + self.onPlayBackStopped() + break + if time.monotonic() > report_t: + report_t += self.REPORT_PERIOD + self._post_event_heartbeat() + logger.debug("Playtime Monitor stopped") + + def initialise(self): + """Initialise play state reports. + + Create an instance ID and post a 'open' event. Subsequent events are to use the + same instance ID. So if posting fails it's no use going on monitoring and post + other events. + + """ + logger.debug("Event OPEN of production %s", self._production_id) + self._instance_id = str(uuid.uuid4()) + data = { + "_v": "1.2.2", + "cid": "e029c040-3119-4192-a219-7086414b5e2b", + "content": self._production_id, + "device": { + "group": "firefox", + "manufacturer": "Firefox", + "model": fetch.USER_AGENT_VERSION, + "os": "Ubuntu", + "userAgent": fetch.USER_AGENT, + "x-codecs": { + "h264_High_3_0": "probably", + "h264_High_3_1": "probably", + "h264_Main_3_0": "probably", + "h264_Main_3_1": "probably", + "hevc": "probably", + "vp9": "probably" + } + }, + "instance": self._instance_id, + "platform": PLATFORM_TAG, + # TODO: playback type short for news-like clips + "playbackType": "vod", + "seq": self._event_seq_nr, + "type": "open", + "user": { + "entitlements": "", + "id": self._user_id + }, + "version": "1.9.64", + "x-playerType": "shaka", + "x-resume": "cross" + } + resp = fetch.web_request('post', EVT_URL, data=data) + if resp.content != b'ok': + logger.warning("Error sending 'open' event for production %s: %s - %s", + self._production_id, resp.status_code, resp.text) + self._status = PlayState.STOPPED + + def _post_event_startup_complete(self): + logger.debug("Event Startup Complete - position %s", self._playtime) + data = {'contentPosStartMillis': self.playtime, + 'initialAudioDescriptionStatus': 'disabled', + 'initialSubStatus': 'disabled', + 'mediaType': 'sting', # news = 'programme', everything else = 'sting' + 'timeTakenMillis': 42} + self._handle_event(data, 'startUpComplete') + + def _post_event_heartbeat(self): + """Post the current play position of a video + + Is to be sent every 30 seconds while the video is playing, or paused. + """ + logger.debug("Event Heartbeat - position %s", self._playtime) + data = {'bitrateKilobitsPerSec': 2399, + 'contentPosMillis': self.playtime} + self._handle_event(data, 'heartbeat') + + def _post_event_seek(self, from_position: float): + """Post a seek event - to be used after skipping forwards or back. + + Not mandatory, currently not used. + + """ + logger.debug("Event Seek - position %s", self._playtime) + + data = {'contentPosFromMillis': from_position, + 'contentPosToMillis': self.playtime, + 'seekButtonInteract': 0} + self._handle_event(data, 'seek') + + + def _post_event_stop(self): + """Stop event. Only seen on mobile app. Currently not used.""" + self._event_seq_nr += 1 + data = { + '_v': '1.2.3', + 'content': self._production_id, + 'data': {}, + 'instance': self._instance_id, + 'platform': PLATFORM_TAG, + 'seq': self._event_seq_nr, + 'type': 'x-stop' + } + fetch.web_request('post', EVT_URL, data=data) + + def _handle_event(self, data:dict, evt_type:str): + self._event_seq_nr += 1 + post_data = { + '_v': '1.2.2', + 'content': self._production_id, + 'data': data, + 'instance': self._instance_id, + 'platform': PLATFORM_TAG, + 'seq': self._event_seq_nr, + 'type': evt_type + } + resp = fetch.web_request('post', EVT_URL, data=post_data) + if resp.content == b'ok': + self._post_errors = 0 + else: + logger.info("Posting progress event failed with HTTP status: %s - %s", resp.status_code, resp.content) + self._post_errors += 1 + if self._post_errors > 3: + # No use going on if event posting continues to fail. + self._status = PlayState.STOPPED + logger.warning("Aborting progress monitoring; more than 3 events have failed.") + + +def playtime_monitor(production_id): + logger.debug("playtime monitor running from thead %s", threading.current_thread().native_id) + try: + player = PlayTimeMonitor(production_id) + player.initialise() + player.wait_until_playing(15) + player.monitor_progress() + except Exception as e: + logger.error("Playtime monitoring aborted due to unhandled exception: %r", e) \ No newline at end of file diff --git a/plugin.video.viwx/resources/settings.xml b/plugin.video.viwx/resources/settings.xml index e171e0a9b..2e0111346 100644 --- a/plugin.video.viwx/resources/settings.xml +++ b/plugin.video.viwx/resources/settings.xml @@ -87,7 +87,7 @@ 0 RunPlugin(plugin://$ID/resources/lib/settings/login) - true + false