chore: historic responses

mure/dtos.py

@@ -2,6 +2,7 @@
 from ssl import SSLContext
 from types import SimpleNamespace
 from typing import Any, Iterable, Literal, Mapping, TypedDict
+from urllib.parse import urlparse
 
 import orjson
 from aiohttp import BasicAuth, ClientTimeout, Fingerprint
@@ -53,6 +54,28 @@ class HTTPResource(Resource):
     method: HTTPMethod
 
 
+@dataclass
+class HistoricResponse:
+    """Historic HTTP response.
+
+    Attributes
+    ----------
+    status : int
+        HTTP status code of response, e.g. 200.
+    reason : str
+        HTTP status reason of response, e.g. "OK".
+    ok : bool
+        Boolean representation of HTTP status code. True if status is <400; otherwise, False.
+    url : str
+        Response URL.
+    """
+
+    status: int
+    reason: str
+    ok: bool
+    url: str
+
+
 @dataclass
 class Response:
     """HTTP response.
@@ -67,12 +90,18 @@ class Response:
         Boolean representation of HTTP status code. True if status is <400; otherwise, False.
     text : str
         Response's body as decoded string.
+    url : str
+        Response URL.
+    history : list[HistoricResponse]
+        List of historic responses (in case requested URL redirected).
     """
 
     status: int
     reason: str
     ok: bool
     text: str
+    url: str
+    history: list[HistoricResponse]
 
     def json(self) -> Any:
         """Deserialize JSON to Python objects.
@@ -98,3 +127,20 @@ def __getitem__(self, attr: str) -> int | str | bool:
             Attribute value.
         """
         return getattr(self, attr)
+
+    def is_same_netloc(self, url: str) -> bool:
+        """True if the given URL has the same netloc as the response URL.
+
+        Parameters
+        ----------
+        url : str
+            URL to check.
+
+        Returns
+        -------
+        bool
+            True if same netloc, False otherwise.
+        """
+        a = urlparse(url)
+        b = urlparse(self.url)
+        return a.netloc.lower() == b.netloc.lower()

mure/iterator.py

@@ -4,7 +4,7 @@
 
 from aiohttp import ClientResponse, ClientSession
 
-from mure.dtos import HTTPResource, Resource, Response
+from mure.dtos import HistoricResponse, HTTPResource, Resource, Response
 from mure.logging import Logger
 
 LOGGER = Logger(__name__)
@@ -136,11 +136,21 @@ async def _process(self, session: ClientSession, resource: HTTPResource) -> Resp
                     reason=response.reason,  # type: ignore
                     ok=response.ok,
                     text=text,
+                    url=response.url.human_repr(),
+                    history=[
+                        HistoricResponse(
+                            historic_response.status,
+                            historic_response.reason,  # type: ignore
+                            historic_response.ok,
+                            historic_response.url.human_repr(),
+                        )
+                        for historic_response in response.history
+                    ],
                 )
         except Exception as error:
             if self._log_errors:
                 LOGGER.error(error)
-            return Response(status=0, reason=repr(error), ok=False, text="")
+            return Response(status=0, reason=repr(error), ok=False, text="", url="", history=[])
 
     @staticmethod
     def chunk(values: Iterable[Any], n: int = 5) -> Iterator[Any]:

tests/test_mure.py

@@ -2,11 +2,11 @@
 from orjson import JSONDecodeError
 
 import mure
-from mure.dtos import Response
+from mure.dtos import HTTPResource, Resource, Response
 
 
 def test_get():
-    resources = [
+    resources: list[Resource] = [
         {"url": "https://httpbin.org/get"},
         {"url": "https://httpbin.org/get", "params": {"foo": "bar"}},
         {"url": "invalid"},
@@ -21,7 +21,7 @@ def test_get():
 
 
 def test_post():
-    resources = [
+    resources: list[Resource] = [
         {"url": "https://httpbin.org/post"},
         {"url": "https://httpbin.org/post", "json": {"foo": "bar"}},
         {"url": "invalid"},
@@ -36,7 +36,7 @@ def test_post():
 
 
 def test_mixed():
-    resources = [
+    resources: list[HTTPResource] = [
         {"method": "GET", "url": "https://httpbin.org/get"},
         {"method": "GET", "url": "https://httpbin.org/get", "params": {"foo": "bar"}},
         {"method": "POST", "url": "https://httpbin.org/post"},