Fixes for 0.2.2 (see desc)

* #58, #61 - Resolved status code 10201 appearing when extracting videos from mobile app share links
* #59 - Suppressed two errors during user video iteration:
    * TypeError: `DeferredItemListIterator` no longer attempts to iterate over a None item_list
    * JSONDecodeError: `DeferredItemListIterator` cuts iteration short if an API request returns null JSON

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tiktokapipy"
-version = "0.2.1"
+version = "0.2.2"
 authors = [
     { name="Russell Newton", email="[email protected]" },
 ]

src/tiktokapipy/api.py

@@ -21,7 +21,7 @@
     VideoPage,
 )
 from tiktokapipy.models.user import User, user_link
-from tiktokapipy.models.video import Video
+from tiktokapipy.models.video import Video, is_mobile_share_link
 from tiktokapipy.util.queries import get_challenge_detail_sync, get_video_detail_sync
 
 _DataModelT = TypeVar("_DataModelT", bound=PrimaryResponseType, covariant=True)
@@ -161,6 +161,34 @@ def video(
         :rtype: :class:`.Video`
         """
         if isinstance(link_or_id, str):
+            if is_mobile_share_link(link_or_id):
+                self.context.clear_cookies()
+                page: Page = self.context.new_page()
+                page.add_init_script(
+                    """
+    if (navigator.webdriver === false) {
+        // Post Chrome 89.0.4339.0 and already good
+    } else if (navigator.webdriver === undefined) {
+        // Pre Chrome 89.0.4339.0 and already good
+    } else {
+        // Pre Chrome 88.0.4291.0 and needs patching
+        delete Object.getPrototypeOf(navigator).webdriver
+    }
+                """
+                )
+
+                def ignore_scripts(route: Route):
+                    if route.request.resource_type == "script":
+                        return route.abort()
+                    return route.continue_()
+
+                page.route("**/*", ignore_scripts)
+                page.goto(link_or_id, wait_until=None)
+                page.wait_for_selector("#SIGI_STATE", state="attached")
+
+                link_or_id = page.url
+
+                page.close()
             video_id = link_or_id.split("/")[-1].split("?")[0]
         else:
             video_id = link_or_id

src/tiktokapipy/async_api.py

@@ -21,7 +21,7 @@
     VideoPage,
 )
 from tiktokapipy.models.user import User, user_link
-from tiktokapipy.models.video import Video
+from tiktokapipy.models.video import Video, is_mobile_share_link
 from tiktokapipy.util.queries import get_challenge_detail_async, get_video_detail_async
 
 _DataModelT = TypeVar("_DataModelT", bound=PrimaryResponseType, covariant=True)
@@ -156,6 +156,34 @@ async def video(
         link_or_id: Union[int, str],
     ) -> Video:
         if isinstance(link_or_id, str):
+            if is_mobile_share_link(link_or_id):
+                await self.context.clear_cookies()
+                page: Page = await self.context.new_page()
+                await page.add_init_script(
+                    """
+    if (navigator.webdriver === false) {
+        // Post Chrome 89.0.4339.0 and already good
+    } else if (navigator.webdriver === undefined) {
+        // Pre Chrome 89.0.4339.0 and already good
+    } else {
+        // Pre Chrome 88.0.4291.0 and needs patching
+        delete Object.getPrototypeOf(navigator).webdriver
+    }
+                """
+                )
+
+                async def ignore_scripts(route: Route):
+                    if route.request.resource_type == "script":
+                        return await route.abort()
+                    return await route.continue_()
+
+                await page.route("**/*", ignore_scripts)
+                await page.goto(link_or_id, wait_until=None)
+                await page.wait_for_selector("#SIGI_STATE", state="attached")
+
+                link_or_id = page.url
+
+                await page.close()
             video_id = link_or_id.split("/")[-1].split("?")[0]
         else:
             video_id = link_or_id

src/tiktokapipy/models/video.py

@@ -274,3 +274,9 @@ def url(self) -> str:
 def video_link(video_id: int) -> str:
     """Get a working link to a TikTok video from the video's unique id."""
     return f"https://m.tiktok.com/v/{video_id}"
+
+
+def is_mobile_share_link(link: str) -> bool:
+    import re
+
+    return re.match(r"https://vm\.tiktok\.com/[0-9A-Za-z]*", link) is not None

src/tiktokapipy/util/deferred_collectors.py

@@ -1,6 +1,8 @@
 import abc
 import time
 import warnings
+from datetime import datetime
+from json import JSONDecodeError
 from typing import AsyncIterator, ForwardRef, Iterator, List, Literal, TypeVar, Union
 
 from playwright.async_api import BrowserContext as AsyncBrowserContext
@@ -175,20 +177,34 @@ def _fetch_sync(self):
         from tiktokapipy.models.raw_data import APIResponse
 
         # noinspection PyTypeChecker
-        raw = make_request_sync(
-            f"{self.from_type}/item_list/",
-            self._cursor,
-            self._target_id,
-            self._api.context,
-            **self._extra_params,
-        )
+        try:
+            raw = make_request_sync(
+                f"{self.from_type}/item_list/",
+                self._cursor,
+                self._target_id,
+                self._api.context,
+                **self._extra_params,
+            )
+        except JSONDecodeError:
+            readable_cursor = (
+                f"video #{self._cursor}"
+                if self.from_type == "challenge"
+                else datetime.fromtimestamp(self._cursor // 1000).strftime("%c")
+            )
+            warnings.warn(
+                f"Unable to grab videos beyond {readable_cursor} (JSONDecodeError), stopping iteration early."
+                f"Try again if you think this is a mistake.",
+                category=TikTokAPIWarning,
+                stacklevel=2,
+            )
+            self._has_more = False
+            raise StopIteration
         converted = APIResponse.model_validate(raw)
-        for item in converted.item_list:
-            item._api = self._api
+        if not converted.item_list:
+            self._has_more = False
+            raise StopIteration
         self._has_more = converted.has_more
         self._cursor = converted.cursor
-        if not converted.item_list:
-            return
 
         for video in converted.item_list:
             try:
@@ -204,20 +220,34 @@ async def _fetch_async(self):
         from tiktokapipy.models.raw_data import APIResponse
 
         # noinspection PyTypeChecker
-        raw = await make_request_async(
-            f"{self.from_type}/item_list/",
-            self._cursor,
-            self._target_id,
-            self._api.context,
-            **self._extra_params,
-        )
+        try:
+            raw = await make_request_async(
+                f"{self.from_type}/item_list/",
+                self._cursor,
+                self._target_id,
+                self._api.context,
+                **self._extra_params,
+            )
+        except JSONDecodeError:
+            readable_cursor = (
+                f"video #{self._cursor}"
+                if self.from_type == "challenge"
+                else datetime.fromtimestamp(self._cursor).strftime("%c")
+            )
+            warnings.warn(
+                f"Unable to grab videos beyond {readable_cursor}, stopping iteration early."
+                f"Try again if you think this is a mistake.",
+                category=TikTokAPIWarning,
+                stacklevel=2,
+            )
+            self._has_more = False
+            raise StopAsyncIteration
         converted = APIResponse.model_validate(raw)
-        for item in converted.item_list:
-            item._api = self._api
+        if not converted.item_list:
+            self._has_more = False
+            raise StopAsyncIteration
         self._has_more = converted.has_more
         self._cursor = converted.cursor
-        if not converted.item_list:
-            return
         for video in converted.item_list:
             try:
                 self._collected_values.append(await self._api.video(video.id))