Skip to content

Commit

Permalink
s3: do not mask failures to retrieve metadata
Browse files Browse the repository at this point in the history
If a file's metadata is being requested and it cannot be found, this
is a real error. We should allow this exception to bubble up, rather
than masking it.
Some backends (e.g. minio) may provide a `KeyCount` value. If this
is present and zero, assume no records are present, and do not
look for the `Contents` list.
  • Loading branch information
nicois committed Nov 29, 2024
1 parent 6a19820 commit 1cfe71b
Showing 1 changed file with 16 additions and 18 deletions.
34 changes: 16 additions & 18 deletions rohmu/object_storage/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,25 +372,23 @@ def iter_key(
if status_code:
self.stats.increase(metric="rohmu.s3.iter_key_response", tags={"status_code": str(status_code)})

for item in response["Contents"]:
if with_metadata:
try:
if response.get("KeyCount") != 0:
for item in response["Contents"]:
if with_metadata:
metadata = {k.lower(): v for k, v in self._metadata_for_key(item["Key"]).items()}
except FileNotFoundFromStorageError:
continue
else:
metadata = None
name = self.format_key_from_backend(item["Key"])
yield IterKeyItem(
type=KEY_TYPE_OBJECT,
value={
"last_modified": item["LastModified"],
"md5": item["ETag"].strip('"'),
"metadata": metadata,
"name": name,
"size": item["Size"],
},
)
else:
metadata = None
name = self.format_key_from_backend(item["Key"])
yield IterKeyItem(
type=KEY_TYPE_OBJECT,
value={
"last_modified": item["LastModified"],
"md5": item["ETag"].strip('"'),
"metadata": metadata,
"name": name,
"size": item["Size"],
},
)

for common_prefix in response.get("CommonPrefixes", []):
yield IterKeyItem(
Expand Down

0 comments on commit 1cfe71b

Please sign in to comment.