Migrate to ruff (#96)

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: '3.12'  # Keep in sync with .readthedocs.yml
+        - python-version: '3.13'  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
         - python-version: '3.13'

.pre-commit-config.yaml

@@ -1,34 +1,7 @@
-default_language_version:
-    python: python3.13
 repos:
-  - hooks:
-      - id: bandit
-        args: [-r, -c, .bandit.yml]
-    repo: https://github.com/PyCQA/bandit
-    rev: 1.8.2
-  - hooks:
-      - id: black
-        language_version: python3
-    repo: https://github.com/ambv/black
-    rev: 24.10.0
-  - hooks:
-      - id: isort
-        language_version: python3
-    repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-  - hooks:
-      - id: flake8
-        language_version: python3
-        additional_dependencies:
-          - flake8-bugbear
-          - flake8-comprehensions
-          - flake8-debugger
-          - flake8-docstrings
-          - flake8-string-format
-    repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
-  - hooks:
-      - id: pyupgrade
-        args: [--py39-plus]
-    repo: https://github.com/asottile/pyupgrade
-    rev: v3.19.1
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.9.7
+  hooks:
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format

.readthedocs.yml

@@ -6,7 +6,7 @@ sphinx:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"  # Keep in sync with .github/workflows/main.yml
+    python: "3.13"  # Keep in sync with .github/workflows/main.yml
 
 python:
   install:

docs/_ext/github.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from __future__ import annotations
 
 from docutils import nodes
 from docutils.parsers.rst.roles import set_classes
@@ -14,11 +14,10 @@ def github_role(
     text,
     lineno,
     inliner,
-    options: Optional[dict] = None,
-    content: Optional[list] = None,
+    options: dict[str, str] | None = None,
+    content=None,
 ):
     options = options or {}
-    content = content or []
     if text.isdigit():
         display_text = f"#{text}"
         url = f"https://github.com/scrapy/itemloaders/issues/{text}"

docs/conf.py

@@ -11,13 +11,13 @@
 # serve to show the default.
 
 import sys
-from os import path
+from pathlib import Path
 
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-sys.path.append(path.dirname(__file__))
-sys.path.insert(0, path.dirname(path.dirname(__file__)))
+sys.path.append(str(Path(__file__).parent))
+sys.path.insert(0, str(Path(__file__).parent.parent))
 
 # General configuration
 # ---------------------

itemloaders/__init__.py

@@ -6,20 +6,21 @@
 
 from __future__ import annotations
 
-from collections.abc import Iterable, MutableMapping
 from contextlib import suppress
-from re import Pattern
 from typing import TYPE_CHECKING, Any, Callable
 
 from itemadapter import ItemAdapter
-from parsel import Selector
+from parsel import Selector  # noqa: TC002  # for sphinx
 from parsel.utils import extract_regex, flatten
 
 from itemloaders.common import wrap_loader_context
 from itemloaders.processors import Identity
 from itemloaders.utils import arg_to_iter
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable, MutableMapping
+    from re import Pattern
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
@@ -135,15 +136,13 @@ def __init__(
     def _values(self) -> dict[str, list[Any]]:
         if self.parent is not None:
             return self.parent._values
-        else:
-            return self._local_values
+        return self._local_values
 
     @property
     def item(self) -> Any:
         if self.parent is not None:
             return self.parent.item
-        else:
-            return self._local_item
+        return self._local_item
 
     def nested_xpath(self, xpath: str, **context: Any) -> Self:
         """
@@ -157,8 +156,7 @@ def nested_xpath(self, xpath: str, **context: Any) -> Self:
         assert self.selector is not None
         selector = self.selector.xpath(xpath)
         context.update(selector=selector)
-        subloader = self.__class__(item=self.item, parent=self, **context)
-        return subloader
+        return self.__class__(item=self.item, parent=self, **context)
 
     def nested_css(self, css: str, **context: Any) -> Self:
         """
@@ -172,8 +170,7 @@ def nested_css(self, css: str, **context: Any) -> Self:
         assert self.selector is not None
         selector = self.selector.css(css)
         context.update(selector=selector)
-        subloader = self.__class__(item=self.item, parent=self, **context)
-        return subloader
+        return self.__class__(item=self.item, parent=self, **context)
 
     def add_value(
         self,
@@ -288,13 +285,13 @@ def get_value(
             if value is None:
                 break
             _proc = proc
-            proc = wrap_loader_context(proc, self.context)
+            proc = wrap_loader_context(proc, self.context)  # noqa: PLW2901
             try:
                 value = proc(value)
             except Exception as e:
                 raise ValueError(
-                    "Error with processor %s value=%r error='%s: %s'"
-                    % (_proc.__class__.__name__, value, type(e).__name__, str(e))
+                    f"Error with processor {_proc.__class__.__name__} "
+                    f"value={value!r} error='{type(e).__name__}: {e!s}'"
                 ) from e
         return value
 
@@ -324,24 +321,24 @@ def get_output_value(self, field_name: str) -> Any:
             return proc(value)
         except Exception as e:
             raise ValueError(
-                "Error with output processor: field=%r value=%r error='%s: %s'"
-                % (field_name, value, type(e).__name__, str(e))
+                f"Error with output processor: field={field_name!r} "
+                f"value={value!r} error='{type(e).__name__}: {e!s}'"
             ) from e
 
     def get_collected_values(self, field_name: str) -> list[Any]:
         """Return the collected values for the given field."""
         return self._values.get(field_name, [])
 
     def get_input_processor(self, field_name: str) -> Callable[..., Any]:
-        proc = getattr(self, "%s_in" % field_name, None)
+        proc = getattr(self, f"{field_name}_in", None)
         if not proc:
             proc = self._get_item_field_attr(
                 field_name, "input_processor", self.default_input_processor
             )
         return unbound_method(proc)
 
     def get_output_processor(self, field_name: str) -> Callable[..., Any]:
-        proc = getattr(self, "%s_out" % field_name, None)
+        proc = getattr(self, f"{field_name}_out", None)
         if not proc:
             proc = self._get_item_field_attr(
                 field_name, "output_processor", self.default_output_processor
@@ -362,22 +359,15 @@ def _process_input_value(self, field_name: str, value: Any) -> Any:
             return proc(value)
         except Exception as e:
             raise ValueError(
-                "Error with input processor %s: field=%r value=%r "
-                "error='%s: %s'"
-                % (
-                    _proc.__class__.__name__,
-                    field_name,
-                    value,
-                    type(e).__name__,
-                    str(e),
-                )
+                f"Error with input processor {_proc.__class__.__name__}:"
+                f" field={field_name!r} value={value!r} error='{type(e).__name__}: {e!s}'"
             ) from e
 
     def _check_selector_method(self) -> None:
         if self.selector is None:
             raise RuntimeError(
-                "To use XPath or CSS selectors, %s "
-                "must be instantiated with a selector" % self.__class__.__name__
+                f"To use XPath or CSS selectors, {self.__class__.__name__} "
+                f"must be instantiated with a selector"
             )
 
     def add_xpath(

itemloaders/common.py

@@ -2,12 +2,14 @@
 
 from __future__ import annotations
 
-from collections.abc import MutableMapping
 from functools import partial
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any, Callable
 
 from itemloaders.utils import get_func_args
 
+if TYPE_CHECKING:
+    from collections.abc import MutableMapping
+
 
 def wrap_loader_context(
     function: Callable[..., Any], context: MutableMapping[str, Any]
@@ -17,5 +19,4 @@ def wrap_loader_context(
     """
     if "loader_context" in get_func_args(function):
         return partial(function, loader_context=context)
-    else:
-        return function
+    return function

itemloaders/processors.py

@@ -7,12 +7,14 @@
 from __future__ import annotations
 
 from collections import ChainMap
-from collections.abc import Iterable, MutableMapping
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any, Callable
 
 from itemloaders.common import wrap_loader_context
 from itemloaders.utils import arg_to_iter
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable, MutableMapping
+
 
 class MapCompose:
     """
@@ -56,7 +58,7 @@ class MapCompose:
     See :class:`Compose` processor for more info.
 
     .. _`parsel selectors`: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.selector.Selector.extract
-    """  # noqa
+    """
 
     def __init__(self, *functions: Callable[..., Any], **default_loader_context: Any):
         self.functions = functions
@@ -79,9 +81,8 @@ def __call__(
                     next_values += arg_to_iter(func(v))
                 except Exception as e:
                     raise ValueError(
-                        "Error in MapCompose with "
-                        "%s value=%r error='%s: %s'"
-                        % (str(func), value, type(e).__name__, str(e))
+                        f"Error in MapCompose with "
+                        f"{func!s} value={value!r} error='{type(e).__name__}: {e!s}'"
                     ) from e
             values = next_values
         return values
@@ -137,9 +138,8 @@ def __call__(
                 value = func(value)
             except Exception as e:
                 raise ValueError(
-                    "Error in Compose with "
-                    "%s value=%r error='%s: %s'"
-                    % (str(func), value, type(e).__name__, str(e))
+                    f"Error in Compose with "
+                    f"{func!s} value={value!r} error='{type(e).__name__}: {e!s}'"
                 ) from e
         return value
 

itemloaders/utils.py

@@ -50,8 +50,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str
                 continue
             args.append(name)
     else:
-        for name in sig.parameters.keys():
-            args.append(name)
+        args = list(sig.parameters)
 
     if stripself and args and args[0] == "self":
         args = args[1:]