Skip to content

Commit

Permalink
fix: add headers compliant with user-agent policy
Browse files Browse the repository at this point in the history
Prior to this commit, when sending several requests in a row,
some requests could fail with `429/Too many requests` as the client
was not providing an `User-Agent` header compliant with Wikimedia's
user-agent policy.

See also: https://meta.wikimedia.org/wiki/User-Agent_policy
  • Loading branch information
marccarre committed Jul 22, 2022
1 parent 66fd002 commit 79dde4b
Showing 1 changed file with 11 additions and 1 deletion.
12 changes: 11 additions & 1 deletion wikidata/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io
import json
import logging
import sys
from typing import (
TYPE_CHECKING,
Callable,
Expand All @@ -21,6 +22,7 @@
import urllib.request
import weakref

from . import __version__
from .cache import CacheKey, CachePolicy, NullCachePolicy
from .entity import Entity, EntityId, EntityType

Expand Down Expand Up @@ -104,7 +106,8 @@ def __init__(self,
None] = None,
entity_type_guess: bool = True,
cache_policy: CachePolicy = NullCachePolicy(),
repr_string: Optional[str] = None) -> None:
repr_string: Optional[str] = None,
user_agent: Optional[str] = None) -> None:
self._using_default_opener = opener is None
if self._using_default_opener:
if urllib.request._opener is None: # type: ignore
Expand All @@ -114,6 +117,13 @@ def __init__(self,
pass
opener = urllib.request._opener # type: ignore
assert isinstance(opener, urllib.request.OpenerDirector)
if not user_agent:
python_version = f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}'
user_agent = f'wikidata-based-bot/{__version__} (https://github.com/dahlia/wikidata) python/{python_version}'
# See also: https://meta.wikimedia.org/wiki/User-Agent_policy
opener.addheaders = {
'User-Agent': user_agent,
}
if datavalue_decoder is None:
from .datavalue import Decoder # noqa: F811
datavalue_decoder = Decoder()
Expand Down

0 comments on commit 79dde4b

Please sign in to comment.