Skip to content

Commit

Permalink
refactor: move arguments to config dataclasses
Browse files Browse the repository at this point in the history
  • Loading branch information
mdwint committed Dec 30, 2023

Verified

This commit was signed with the committer’s verified signature.
W-Mai Benign X
1 parent f2b96a9 commit ee4dfc4
Showing 6 changed files with 103 additions and 76 deletions.
28 changes: 24 additions & 4 deletions s3pypi/__main__.py
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ def get_arg_parser():
p.add_argument(
"--s3-put-args",
type=string_dict,
default={},
help=(
"Optional extra arguments to S3 PutObject calls. Example: "
"'ServerSideEncryption=aws:kms,SSEKMSKeyId=1234...'"
@@ -73,12 +74,31 @@ def get_arg_parser():
return p


def main(*args):
kwargs = vars(get_arg_parser().parse_args(args or sys.argv[1:]))
log.setLevel(logging.DEBUG if kwargs.pop("verbose") else logging.INFO)
def main(*raw_args: str) -> None:
args = get_arg_parser().parse_args(raw_args or sys.argv[1:])
log.setLevel(logging.DEBUG if args.verbose else logging.INFO)

cfg = core.Config(
dist=args.dist,
s3=core.S3Config(
bucket=args.bucket,
prefix=args.prefix,
endpoint_url=args.s3_endpoint_url,
put_kwargs=args.s3_put_args,
unsafe_s3_website=args.unsafe_s3_website,
no_sign_request=args.no_sign_request,
),
force=args.force,
lock_indexes=args.lock_indexes,
put_root_index=args.put_root_index,
profile=args.profile,
region=args.region,
)
if args.acl:
cfg.s3.put_kwargs["ACL"] = args.acl

try:
core.upload_packages(**kwargs)
core.upload_packages(cfg)
except core.S3PyPiError as e:
sys.exit(f"ERROR: {e}")

38 changes: 20 additions & 18 deletions s3pypi/core.py
Original file line number Diff line number Diff line change
@@ -15,13 +15,24 @@
from s3pypi.exceptions import S3PyPiError
from s3pypi.index import Hash
from s3pypi.locking import DummyLocker, DynamoDBLocker
from s3pypi.storage import S3Storage
from s3pypi.storage import S3Config, S3Storage

log = logging.getLogger(__prog__)

PackageMetadata = email.message.Message


@dataclass
class Config:
dist: List[Path]
s3: S3Config
force: bool = False
lock_indexes: bool = False
put_root_index: bool = False
profile: Optional[str] = None
region: Optional[str] = None


@dataclass
class Distribution:
name: str
@@ -33,25 +44,16 @@ def normalize_package_name(name: str) -> str:
return re.sub(r"[-_.]+", "-", name.lower())


def upload_packages(
dist: List[Path],
bucket: str,
force: bool = False,
lock_indexes: bool = False,
put_root_index: bool = False,
profile: Optional[str] = None,
region: Optional[str] = None,
**kwargs,
):
session = boto3.Session(profile_name=profile, region_name=region)
storage = S3Storage(session, bucket, **kwargs)
def upload_packages(cfg: Config) -> None:
session = boto3.Session(profile_name=cfg.profile, region_name=cfg.region)
storage = S3Storage(session, cfg.s3)
lock = (
DynamoDBLocker(session, table=f"{bucket}-locks")
if lock_indexes
DynamoDBLocker(session, table=f"{cfg.s3.bucket}-locks")
if cfg.lock_indexes
else DummyLocker()
)

distributions = parse_distributions(dist)
distributions = parse_distributions(cfg.dist)
get_name = attrgetter("name")

for name, group in groupby(sorted(distributions, key=get_name), get_name):
@@ -62,7 +64,7 @@ def upload_packages(
for distr in group:
filename = distr.local_path.name

if not force and filename in index.filenames:
if not cfg.force and filename in index.filenames:
msg = "%s already exists! (use --force to overwrite)"
log.warning(msg, filename)
else:
@@ -72,7 +74,7 @@ def upload_packages(

storage.put_index(directory, index)

if put_root_index:
if cfg.put_root_index:
with lock(storage.root):
index = storage.build_root_index()
storage.put_index(storage.root, index)
52 changes: 24 additions & 28 deletions s3pypi/storage.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

@@ -8,41 +9,36 @@
from s3pypi.index import Index


@dataclass
class S3Config:
bucket: str
prefix: Optional[str] = None
endpoint_url: Optional[str] = None
put_kwargs: dict = field(default_factory=dict)
unsafe_s3_website: bool = False
no_sign_request: bool = False


class S3Storage:
root = "/"
_index = "index.html"

def __init__(
self,
session: boto3.session.Session,
bucket: str,
prefix: Optional[str] = None,
acl: Optional[str] = None,
s3_endpoint_url: Optional[str] = None,
s3_put_args: Optional[dict] = None,
unsafe_s3_website: bool = False,
no_sign_request: bool = False,
):
def __init__(self, session: boto3.session.Session, cfg: S3Config):
_config = None
if no_sign_request:
if cfg.no_sign_request:
_config = Config(signature_version=botocore.session.UNSIGNED)

self.s3 = session.resource("s3", endpoint_url=s3_endpoint_url, config=_config)
self.bucket = bucket
self.prefix = prefix
self.index_name = self._index if unsafe_s3_website else ""
self.put_kwargs = dict(
ACL=acl or "private",
**(s3_put_args or {}),
)
self.s3 = session.resource("s3", endpoint_url=cfg.endpoint_url, config=_config)
self.index_name = self._index if cfg.unsafe_s3_website else ""
self.cfg = cfg

def _object(self, directory: str, filename: str):
parts = [directory, filename]
if parts == [self.root, self.index_name]:
parts = [self._index]
if self.prefix:
parts.insert(0, self.prefix)
return self.s3.Object(self.bucket, key="/".join(parts))
if self.cfg.prefix:
parts.insert(0, self.cfg.prefix)
return self.s3.Object(self.cfg.bucket, key="/".join(parts))

def get_index(self, directory: str) -> Index:
try:
@@ -54,11 +50,11 @@ def get_index(self, directory: str) -> Index:
def build_root_index(self) -> Index:
paginator = self.s3.meta.client.get_paginator("list_objects_v2")
result = paginator.paginate(
Bucket=self.bucket,
Prefix=self.prefix or "",
Bucket=self.cfg.bucket,
Prefix=self.cfg.prefix or "",
Delimiter="/",
)
n = len(self.prefix) + 1 if self.prefix else 0
n = len(self.cfg.prefix) + 1 if self.cfg.prefix else 0
dirs = (p.get("Prefix")[n:] for p in result.search("CommonPrefixes"))
return Index(dict.fromkeys(dirs))

@@ -67,13 +63,13 @@ def put_index(self, directory: str, index: Index):
Body=index.to_html(),
ContentType="text/html",
CacheControl="public, must-revalidate, proxy-revalidate, max-age=0",
**self.put_kwargs,
**self.cfg.put_kwargs,
)

def put_distribution(self, directory: str, local_path: Path):
with open(local_path, mode="rb") as f:
self._object(directory, local_path.name).put(
Body=f,
ContentType="application/x-gzip",
**self.put_kwargs,
**self.cfg.put_kwargs,
)
28 changes: 28 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[tool:pytest]
addopts =
--tb=short

[flake8]
max-line-length = 80
max-complexity = 18
exclude = .tox/ .venv/ build/ dist/
select = B,C,E,F,W,T4,B9
ignore = E203,E501,W503
show_source = True

[isort]
line_length = 88
multi_line_output = 3
include_trailing_comma = True
force_grid_wrap = 0
combine_as_imports = True
default_section = THIRDPARTY
known_first_party = s3pypi,tests,handler

[mypy]
warn_redundant_casts = True
warn_unused_ignores = True
warn_unreachable = True

[mypy-s3pypi.*]
disallow_untyped_defs = True
12 changes: 7 additions & 5 deletions tests/integration/test_storage.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
from s3pypi.index import Index
from s3pypi.storage import S3Storage
from s3pypi.storage import S3Config, S3Storage


def test_index_storage_roundtrip(boto3_session, s3_bucket):
directory = "foo"
index = Index({"bar": None})

storage = S3Storage(boto3_session, s3_bucket.name)
cfg = S3Config(bucket=s3_bucket.name)
storage = S3Storage(boto3_session, cfg)

storage.put_index(directory, index)
got = storage.get_index(directory)

assert got == index


def test_prefix_in_s3_key(boto3_session):
prefix = "1234567890"
cfg = S3Config(bucket="example", prefix="1234567890")
storage = S3Storage(boto3_session, cfg)

storage = S3Storage(boto3_session, bucket="example", prefix=prefix)
obj = storage._object(directory="foo", filename="bar")

assert obj.key.startswith(prefix + "/")
assert obj.key.startswith(cfg.prefix + "/")
21 changes: 0 additions & 21 deletions tox.ini
Original file line number Diff line number Diff line change
@@ -38,24 +38,3 @@ commands =
flake8
black --check --diff .
isort --check-only .

[tool:pytest]
addopts =
--tb=short

[flake8]
max-line-length = 80
max-complexity = 18
exclude = .tox/ .venv/ build/ dist/
select = B,C,E,F,W,T4,B9
ignore = E203,E501,W503
show_source = True

[isort]
line_length = 88
multi_line_output = 3
include_trailing_comma = True
force_grid_wrap = 0
combine_as_imports = True
default_section = THIRDPARTY
known_first_party = s3pypi,tests,handler

0 comments on commit ee4dfc4

Please sign in to comment.