Skip to content

Commit

Permalink
s3: do not mask failures to retrieve metadata
Browse files Browse the repository at this point in the history
If a file's metadata is being requested and it cannot be found, this
is a real error. We should allow this exception to bubble up, rather
than masking it.
Some backends (e.g. minio) may provide a `KeyCount` value. If this
is present and zero, assume no records are present, and do not
look for the `Contents` list.
  • Loading branch information
nicois committed Dec 8, 2024
1 parent 6a19820 commit 0e3639a
Showing 1 changed file with 17 additions and 19 deletions.
36 changes: 17 additions & 19 deletions rohmu/object_storage/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,25 +372,23 @@ def iter_key(
if status_code:
self.stats.increase(metric="rohmu.s3.iter_key_response", tags={"status_code": str(status_code)})

for item in response["Contents"]:
if with_metadata:
try:
if response.get("KeyCount") != 0:
for item in response["Contents"]:
if with_metadata:
metadata = {k.lower(): v for k, v in self._metadata_for_key(item["Key"]).items()}
except FileNotFoundFromStorageError:
continue
else:
metadata = None
name = self.format_key_from_backend(item["Key"])
yield IterKeyItem(
type=KEY_TYPE_OBJECT,
value={
"last_modified": item["LastModified"],
"md5": item["ETag"].strip('"'),
"metadata": metadata,
"name": name,
"size": item["Size"],
},
)
else:
metadata = None
name = self.format_key_from_backend(item["Key"])
yield IterKeyItem(
type=KEY_TYPE_OBJECT,
value={
"last_modified": item["LastModified"],
"md5": item["ETag"].strip('"'),
"metadata": metadata,
"name": name,
"size": item["Size"],
},
)

for common_prefix in response.get("CommonPrefixes", []):
yield IterKeyItem(
Expand All @@ -408,7 +406,7 @@ def _get_object_stream(self, key: str, byte_range: Optional[tuple[int, int]]) ->
kwargs: dict[str, Any] = {}
if byte_range:
kwargs["Range"] = f"bytes={byte_range[0]}-{byte_range[1]}"
status_code : int|None = None
status_code: int | None = None
try:
# Actual usage is accounted for in
# _read_object_to_fileobj, although that omits the initial
Expand Down

0 comments on commit 0e3639a

Please sign in to comment.