diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e1f55b34..eb3578c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: uses: actions/setup-python@v4 - id: prepare-sphinx - run: pip install sphinx==4.5.0 sphinx-rtd-theme==1.0.0 + run: pip install -r pr/requirements.docs.txt - id: run-sphinx run: sphinx-build -b html pr/docs gh-pages -E -d $GITHUB_WORKSPACE/.doctree diff --git a/.github/workflows/publish-pypi.yaml b/.github/workflows/publish-pypi.yaml index 583ab678..1ec5b45c 100644 --- a/.github/workflows/publish-pypi.yaml +++ b/.github/workflows/publish-pypi.yaml @@ -42,6 +42,6 @@ jobs: test $(git for-each-ref --format='%(objecttype)' ${GITHUB_REF}) == tag - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 32d1d3c9..2efb51a7 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -31,7 +31,7 @@ jobs: uses: actions/setup-python@v4 - name: "Install Sphinx & Theme" - run: pip install sphinx==4.5.0 sphinx-rtd-theme==1.0.0 + run: pip install -r main/requirements.docs.txt - name: "Run Sphinx" run: sphinx-build -b html main/docs gh-pages -E -d $GITHUB_WORKSPACE/.doctree diff --git a/Makefile b/Makefile index b6a36882..987684e1 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ long_ver = $(shell git describe --long 2>/dev/null | cut -f2 -d/ || echo $(short PYTHON ?= python3 PYTHON_SOURCE_DIRS = rohmu/ test/ PYTEST_ARG ?= -v +RST_FILES = README.rst docs/about.rst docs/autodoc.rst docs/development.rst docs/index.rst docs/toc.rst docs/usage.rst .PHONY: fedora-dev-setup fedora-dev-setup: @@ -37,7 +38,7 @@ mypy: fmt: isort $(PYTHON_SOURCE_DIRS) black $(PYTHON_SOURCE_DIRS) - $(PYTHON) -m rstfmt README.rst -w 100 + $(PYTHON) -m rstfmt $(RST_FILES) -w 100 .PHONY: coverage coverage: diff --git a/README.rst b/README.rst index ee982dc0..8c47e2f1 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,10 @@ +####### + Rohmu +####### + +.. + start-include-intro + |Rohmu logo| |Build badge| |PyPI badge| |Python versions badge| @@ -8,36 +15,82 @@ supports main public clouds such as GCP, AWS, and Azure for backup storage. Rohm backup tools such as PGHoard_ for PostgreSQL, MyHoard_ for MySQL, and Astacus_ for M3, ClickHouse, and other databases. -########## +.. + end-include-intro + +.. + start-include-features + +********** Features -########## +********** - Supported object storages: Azure, GCP, S3, Swift (OpenStack), local file storage, and SFTP. - Supported compression algorithms: Snappy_, zstd_, and lzma_. -############## +.. + end-include-features + +.. + start-include-requirements + +************** Requirements -############## +************** Rohmu requires Python >= 3.8. For Python library dependencies, refer to setup.cfg_. -############### +.. + end-include-requirements + +.. + start-include-usage + +*************** Usage example -############### +*************** *Add usage example here* For real-world usage, see `how Rohmu is used in PGHoard`_. -############# +.. + end-include-usage + +************* Development -############# +************* *TODO* -######### +.. + start-include-building-the-package + +********************** + Building the package +********************** + +To build an installation package for your distribution, go to the root directory of a Rohmu Git +checkout and run: + +Fedora: + +.. code:: + + sudo make fedora-dev-setup + make rpm + +This will produce a ``.rpm`` package usually into ``rpm/RPMS/noarch/``. + +.. + end-include-building-the-package + +.. + start-include-license + +********* License -######### +********* Rohmu is licensed under the Apache license, version 2.0. Full license text is available in the LICENSE_ file. @@ -45,17 +98,23 @@ LICENSE_ file. Please note that the project explicitly does not require a CLA (Contributor License Agreement) from its contributors. -############ +.. + end-include-license + +.. + start-include-trademarks-and-credits + +************ Trademarks -############ +************ PostgreSQL, MySQL, M3 and ClickHouse are trademarks and property of their respective owners. All product and service names used in this website are for identification purposes only and do not imply endorsement. -######### +********* Credits -######### +********* Rohmu was created by and is maintained by Aiven_. @@ -63,20 +122,38 @@ Rohmu was originally a part of PGHoard_ but was later extracted to its own GitHu The Rohmu logo was created by `@evche-aiven`_. -######### +.. + end-include-trademarks-and-credits + +.. + start-include-contact + +********* Contact -######### +********* Bug reports and patches are very welcome; please post them as GitHub issues and pull requests at rohmu_repo_. To report any possible vulnerabilities or other serious issues, please see our security_ policy. -########### +.. + end-include-contact + +.. + start-include-copyright + +*********** Copyright -########### +*********** Copyright (C) 2023 Aiven Ltd and contributors to the Rohmu project. +.. + end-include-copyright + +.. + start-include-links + .. --------- Links --------- diff --git a/docs/about.rst b/docs/about.rst index 4b1bd92e..93bb333a 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -1,12 +1,10 @@ -About Rohmu -=========== +############# + About Rohmu +############# -Features --------- - -* Supported object storages: Azure, GCP, S3, Swift (OpenStack), local - file storage and SFTP. -* Supported compression algorithms: Snappy, - `zstd `__ and - `lzma `__. +.. include:: ../README.rst + :start-after: start-include-features + :end-before: end-include-features +.. include:: ../README.rst + :start-after: start-include-links diff --git a/docs/api__object_storage.rst b/docs/api__object_storage.rst new file mode 100644 index 00000000..d7002f44 --- /dev/null +++ b/docs/api__object_storage.rst @@ -0,0 +1,61 @@ + + +Azure +===== + +.. autopydantic_model:: rohmu.object_storage.config.AzureObjectStorageConfig + +.. autoclass:: rohmu.object_storage.azure.AzureTransfer + :members: + + +Google +====== + +.. autopydantic_model:: rohmu.object_storage.config.GoogleObjectStorageConfig + +.. autoclass:: rohmu.object_storage.google.GoogleTransfer + :members: + +.. autoclass:: rohmu.object_storage.google.MediaStreamUpload + :members: + +.. autoclass:: rohmu.object_storage.google.MediaIoBaseDownloadWithByteRange + :members: + + +Local +===== + +.. autopydantic_model:: rohmu.object_storage.config.LocalObjectStorageConfig + +.. autoclass:: rohmu.object_storage.local.LocalTransfer + :members: + + +S3 +== + +.. autopydantic_model:: rohmu.object_storage.config.S3ObjectStorageConfig + +.. autoclass:: rohmu.object_storage.s3.S3Transfer + :members: + + +SFTP +==== + +.. autopydantic_model:: rohmu.object_storage.config.SFTPObjectStorageConfig + +.. autoclass:: rohmu.object_storage.sftp.SFTPTransfer + :members: + + +Swift +===== + +.. autopydantic_model:: rohmu.object_storage.config.SwiftObjectStorageConfig + +.. autoclass:: rohmu.object_storage.swift.SwiftTransfer + :members: + diff --git a/docs/autodoc.rst b/docs/autodoc.rst new file mode 100644 index 00000000..a699237c --- /dev/null +++ b/docs/autodoc.rst @@ -0,0 +1,87 @@ +########################## + Python API Documentation +########################## + +******** + Common +******** + +.. autoclass:: rohmu.BaseTransfer + +.. autofunction:: rohmu.get_class_for_notifier + +.. autofunction:: rohmu.get_class_for_storage_driver + +.. autofunction:: rohmu.get_class_for_transfer + +.. autofunction:: rohmu.get_notifier + +.. autofunction:: rohmu.get_transfer_from_model + +.. autofunction:: rohmu.get_transfer_model + +.. autofunction:: rohmu.get_transfer + +.. autoclass:: rohmu.Notifier + +.. autoclass:: rohmu.ProxyInfo + +.. autodata:: rohmu.S3AddressingStyle + +.. autoclass:: rohmu.StorageDriver + +.. autoclass:: rohmu.StorageModel + +*************** + Delta Backups +*************** + +.. automodule:: rohmu.delta + :members: + +.. automodule:: rohmu.delta.common + :members: + +.. automodule:: rohmu.delta.snapshot + :members: + +*********** + Notifiers +*********** + +.. automodule:: rohmu.notifier + :members: + +.. automodule:: rohmu.notifier.interface + :members: + +HTTP Notifier +============= + +.. automodule:: rohmu.notifier.http + :members: + +Logger Notifier +=============== + +.. automodule:: rohmu.notifier.logger + :members: + +Null Notifier +============= + +.. automodule:: rohmu.notifier.null + :members: + +***************** + Object Storages +***************** + +.. include:: api__object_storage.rst + +************ + Exceptions +************ + +.. automodule:: rohmu.errors + :members: diff --git a/docs/conf.py b/docs/conf.py index e9f0a410..cd7eb6e0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,20 +15,21 @@ sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- +from rohmu.version import VERSION project = "Rohmu" -copyright = "2022, Aiven" +copyright = "2023, Aiven" author = "Aiven" # The full version, including alpha/beta/rc tags -release = "1.1.2" +release = VERSION # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx_rtd_theme"] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx_rtd_theme", "sphinxcontrib.autodoc_pydantic"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/development.rst b/docs/development.rst index a5b57731..6c2cd0d5 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,23 +1,14 @@ -Development -============ +############# + Development +############# -Requirements ------------- +.. include:: ../README.rst + :start-after: start-include-requirements + :end-before: end-include-requirements -Rohmu requires Python >= 3.8. For Python library dependencies, have a -look at -`requirements.txt `__. +.. include:: ../README.rst + :start-after: start-include-building-the-package + :end-before: end-include-building-the-package - -Building the package --------------------- - -To build an installation package for your distribution, go to the root -directory of a Rohmu Git checkout and run: - -Fedora:: - - sudo make fedora-dev-setup - make rpm - -This will produce a ``.rpm`` package usually into ``rpm/RPMS/noarch/``. +.. include:: ../README.rst + :start-after: start-include-links diff --git a/docs/index.rst b/docs/index.rst index 1489b593..ecb1a2ec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,72 +1,28 @@ -Rohmu -======= +####### + Rohmu +####### -|BuildStatus| +.. include:: toc.rst -.. |BuildStatus| image:: https://github.com/aiven/rohmu/actions/workflows/build.yml/badge.svg?branch=main - :target: https://github.com/aiven/rohmu/actions - :alt: Build Status +.. include:: ../README.rst + :start-after: start-include-intro + :end-before: end-include-intro -``Rohmu`` is a Python library for building backup tools for databases -providing functionality for compression, encryption and transferring -data between the database and an object storage. Rohmu supports main -public clouds such as GCP, AWS and Azure for backup storage. Rohmu is -used in various backup tools such as -`PGHoard `__ for PostgreSQL, -`MyHoard `__ for MySQL and -`Astacus `__ for M3, ClickHouse and -other databases. +.. include:: ../README.rst + :start-after: start-include-license + :end-before: end-include-license +.. include:: ../README.rst + :start-after: start-include-trademarks-and-credits + :end-before: end-include-trademarks-and-credits -.. toctree:: - :maxdepth: 2 - :caption: Contents - :glob: - :hidden: +.. include:: ../README.rst + :start-after: start-include-contact + :end-before: end-include-contact - about - usage - development +.. include:: ../README.rst + :start-after: start-include-copyright + :end-before: end-include-copyright -License -======= - -Rohmu is licensed under the Apache license, version 2.0. Full license -text is available in the `LICENSE `__ file. - -Please note that the project explicitly does not require a CLA -(Contributor License Agreement) from its contributors. - -Trademarks -========== - -PostgreSQL, MySQL, M3 and ClickHouse are trademarks and property of -their respective owners. All product and service names used in this -website are for identification purposes only and do not imply -endorsement. - -Credits -======= - -Rohmu was created by and is maintained by `Aiven -`__. - -Rohmu was originally a part of `PGHoard -`__ but was later extracted to its -own GitHub project. - -The Rohmu logo was created by `@evche-aiven -`__. - -Contact -======= - -Bug reports and patches are very welcome, please post them as GitHub -issues and pull requests at https://github.com/aiven/rohmu . To report -any possible vulnerabilities or other serious issues please see our -`security `__ policy. - -Copyright -========= - -Copyright (C) 2022 Aiven Ltd and contributors to the Rohmu project. +.. include:: ../README.rst + :start-after: start-include-links diff --git a/docs/toc.rst b/docs/toc.rst new file mode 100644 index 00000000..8c25893d --- /dev/null +++ b/docs/toc.rst @@ -0,0 +1,9 @@ +.. toctree:: + :maxdepth: 2 + :caption: Contents + :hidden: + + autodoc + about + usage + development diff --git a/docs/usage.rst b/docs/usage.rst index 8b350899..9494e132 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,5 +1,10 @@ -Usage example -============= +####### + Usage +####### -For real-world usage you can have a look at `how Rohmu is used in -PGHoard `__. +.. include:: ../README.rst + :start-after: start-include-usage + :end-before: end-include-usage + +.. include:: ../README.rst + :start-after: start-include-links diff --git a/requirements.docs.txt b/requirements.docs.txt new file mode 100644 index 00000000..dc2134a2 --- /dev/null +++ b/requirements.docs.txt @@ -0,0 +1,3 @@ +sphinx==5.3.0 +sphinx-rtd-theme==1.0.0 +autodoc-pydantic==1.9.0 # upgrade to 2.x when rohmu migrates to Pydantic 2.x diff --git a/rohmu.spec b/rohmu.spec index a3e43496..756e2b60 100644 --- a/rohmu.spec +++ b/rohmu.spec @@ -1,7 +1,7 @@ Name: python3-rohmu Version: %{major_version} Release: %{minor_version}%{?dist} -Url: https://github.com/aiven/rohmu +Url: https://github.com/Aiven-Open/rohmu Summary: Object storage encryption and compression library License: ASL 2.0 Source0: rohmu-rpm-src.tar diff --git a/rohmu/__init__.py b/rohmu/__init__.py index 172a07b0..71427c20 100644 --- a/rohmu/__init__.py +++ b/rohmu/__init__.py @@ -1,10 +1,8 @@ -""" -rohmu +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu""" -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" from .common.constants import IO_BLOCK_SIZE from .common.models import StorageModel from .errors import InvalidConfigurationError diff --git a/rohmu/common/statsd.py b/rohmu/common/statsd.py index 7ea1eda7..613cf5ad 100644 --- a/rohmu/common/statsd.py +++ b/rohmu/common/statsd.py @@ -1,8 +1,6 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details - -StatsD client +# Copyright (c) 2020 Aiven Ltd +# See LICENSE for details +"""StatsD client Supports telegraf's statsd protocol extension for 'key=value' tags: @@ -12,7 +10,6 @@ - pghoard base (pghoard.metrics.statsd) - myhoard timing_manager method - pydantic configuration + explicit typing - """ from __future__ import annotations diff --git a/rohmu/common/strenum.py b/rohmu/common/strenum.py index ed8da07b..fe9e1a0e 100644 --- a/rohmu/common/strenum.py +++ b/rohmu/common/strenum.py @@ -1,9 +1,6 @@ -""" -rohmu - StrEnum - -Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""rohmu - StrEnum""" from __future__ import annotations from typing import Optional diff --git a/rohmu/compressor.py b/rohmu/compressor.py index f8c88c53..d90cd7b5 100644 --- a/rohmu/compressor.py +++ b/rohmu/compressor.py @@ -1,9 +1,6 @@ -""" -rohmu - compressor interface - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - compressor interface""" from .errors import InvalidConfigurationError from .filewrap import Sink, Stream from .snappyfile import SnappyFile diff --git a/rohmu/dates.py b/rohmu/dates.py index 7656ac7a..1d3843a1 100644 --- a/rohmu/dates.py +++ b/rohmu/dates.py @@ -1,9 +1,6 @@ -""" -rohmu - timestamp handling - -Copyright (c) 2017 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2017 Ohmu Ltd +# See LICENSE for details +"""Rohmu - timestamp handling""" import datetime import dateutil.parser import dateutil.tz diff --git a/rohmu/encryptor.py b/rohmu/encryptor.py index 1e3e0a22..23d4de9c 100644 --- a/rohmu/encryptor.py +++ b/rohmu/encryptor.py @@ -1,3 +1,5 @@ +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details """ rohmu - content encryption diff --git a/rohmu/errors.py b/rohmu/errors.py index 72175043..b513d675 100644 --- a/rohmu/errors.py +++ b/rohmu/errors.py @@ -1,9 +1,6 @@ -""" -rohmu - exception classes - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - exception classes""" class Error(Exception): diff --git a/rohmu/filewrap.py b/rohmu/filewrap.py index cbfafdf2..fd9bab6d 100644 --- a/rohmu/filewrap.py +++ b/rohmu/filewrap.py @@ -1,9 +1,6 @@ -""" -rohmu - file transformation wrapper - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - file transformation wrapper""" from .errors import UninitializedError from .typing import BinaryData, FileLike, HasRead, HasWrite from typing import Callable, Optional diff --git a/rohmu/inotify.py b/rohmu/inotify.py index 58a02e6c..be78cb95 100644 --- a/rohmu/inotify.py +++ b/rohmu/inotify.py @@ -1,9 +1,6 @@ -""" -rohmu - inotify wrapper - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - inotify wrapper""" from __future__ import annotations diff --git a/rohmu/notifier/__init__.py b/rohmu/notifier/__init__.py index 31e07b40..9918d7a1 100644 --- a/rohmu/notifier/__init__.py +++ b/rohmu/notifier/__init__.py @@ -1 +1 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ diff --git a/rohmu/notifier/http.py b/rohmu/notifier/http.py index ba6ac875..22a1e910 100644 --- a/rohmu/notifier/http.py +++ b/rohmu/notifier/http.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations from .interface import Notifier diff --git a/rohmu/notifier/interface.py b/rohmu/notifier/interface.py index a0eee6fb..94839d93 100644 --- a/rohmu/notifier/interface.py +++ b/rohmu/notifier/interface.py @@ -1,4 +1,5 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ + from __future__ import annotations from abc import ABC, abstractmethod diff --git a/rohmu/notifier/logger.py b/rohmu/notifier/logger.py index 0fc70557..56d28417 100644 --- a/rohmu/notifier/logger.py +++ b/rohmu/notifier/logger.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations diff --git a/rohmu/notifier/null.py b/rohmu/notifier/null.py index e16e22d4..8463f623 100644 --- a/rohmu/notifier/null.py +++ b/rohmu/notifier/null.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations diff --git a/rohmu/object_storage/__init__.py b/rohmu/object_storage/__init__.py index a9150144..deb771cf 100644 --- a/rohmu/object_storage/__init__.py +++ b/rohmu/object_storage/__init__.py @@ -1,7 +1,4 @@ -""" -rohmu - object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - object store interface""" diff --git a/rohmu/object_storage/azure.py b/rohmu/object_storage/azure.py index 1096dc93..b3ecd8ee 100644 --- a/rohmu/object_storage/azure.py +++ b/rohmu/object_storage/azure.py @@ -1,10 +1,7 @@ -""" -rohmu - azure object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - azure object store interface""" from __future__ import annotations diff --git a/rohmu/object_storage/base.py b/rohmu/object_storage/base.py index 27b33b58..5d97691c 100644 --- a/rohmu/object_storage/base.py +++ b/rohmu/object_storage/base.py @@ -1,10 +1,7 @@ -""" -rohmu - object_storage.base - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - object_storage.base""" from __future__ import annotations from contextlib import suppress diff --git a/rohmu/object_storage/config.py b/rohmu/object_storage/config.py index 9f327677..89554a23 100644 --- a/rohmu/object_storage/config.py +++ b/rohmu/object_storage/config.py @@ -1,10 +1,7 @@ -""" -rohmu - azure object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - azure object store interface""" from __future__ import annotations diff --git a/rohmu/object_storage/google.py b/rohmu/object_storage/google.py index 467a3c88..41352c57 100644 --- a/rohmu/object_storage/google.py +++ b/rohmu/object_storage/google.py @@ -1,10 +1,7 @@ -""" -rohmu - google cloud object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - google cloud object store interface""" from __future__ import annotations @@ -126,15 +123,16 @@ def base64_to_hex(b64val: Union[str, bytes]) -> str: @dataclasses.dataclass class Reporter: - """Used for storing default and also reporting them accordingly + """Used for storing default and also reporting them accordingly. + + The whole point of this class is to handle different cases: - the whole point of this class is to handle different cases - - when file size is too small, reporting *_CHUNK_SIZE is wrong as 1000's of - small files over a long period (may be a month) is a lot of error - - Same is the case of _retry_on_reset - - This error extrapolates when replication is being used + - When file size is too small, reporting ``*_CHUNK_SIZE`` is wrong as 1000's of + small files over a long period (may be a month) is a lot of error; + - Same is the case of _retry_on_reset; + - This error extrapolates when replication is being used. - size should ideally be min(real_size, *_CHUNK_SIZE). real_size is avaiable + Size should ideally be min(real_size, ``*_CHUNK_SIZE``). real_size is avaiable when download or upload (from string or file), but when using FD, size is not known so assuming something too big for small files is almost wrong and in case it is bigger than CHUNKED then status reporting (from the diff --git a/rohmu/object_storage/local.py b/rohmu/object_storage/local.py index 78c2a608..4be71a29 100644 --- a/rohmu/object_storage/local.py +++ b/rohmu/object_storage/local.py @@ -1,10 +1,7 @@ -""" -rohmu - local filesystem interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - local filesystem interface""" from __future__ import annotations diff --git a/rohmu/object_storage/s3.py b/rohmu/object_storage/s3.py index ecdcb765..f81d1ae7 100644 --- a/rohmu/object_storage/s3.py +++ b/rohmu/object_storage/s3.py @@ -1,10 +1,7 @@ -""" -rohmu - aws s3 object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - aws s3 object store interface""" from __future__ import annotations diff --git a/rohmu/object_storage/sftp.py b/rohmu/object_storage/sftp.py index 3d666295..985c5f2c 100644 --- a/rohmu/object_storage/sftp.py +++ b/rohmu/object_storage/sftp.py @@ -1,10 +1,7 @@ -""" -rohmu - sftp object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - sftp object store interface""" from io import BytesIO from rohmu.common.statsd import StatsdConfig diff --git a/rohmu/object_storage/swift.py b/rohmu/object_storage/swift.py index 9bd0f76b..b0b98ac8 100644 --- a/rohmu/object_storage/swift.py +++ b/rohmu/object_storage/swift.py @@ -1,10 +1,7 @@ -""" -rohmu - openstack swift object store interface - -Copyright (c) 2016 Ohmu Ltd -Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - openstack swift object store interface""" from __future__ import annotations diff --git a/rohmu/rohmufile.py b/rohmu/rohmufile.py index b0777103..98a9e2de 100644 --- a/rohmu/rohmufile.py +++ b/rohmu/rohmufile.py @@ -1,9 +1,6 @@ -""" -rohmu - rohmu data transformation interface - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - rohmu data transformation interface""" from __future__ import annotations diff --git a/rohmu/snappyfile.py b/rohmu/snappyfile.py index 053c0742..1dd0ca68 100644 --- a/rohmu/snappyfile.py +++ b/rohmu/snappyfile.py @@ -1,9 +1,6 @@ -""" -rohmu - file-like interface for snappy - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - file-like interface for snappy""" from .common.constants import IO_BLOCK_SIZE from .filewrap import FileWrap from .typing import BinaryData, FileLike diff --git a/rohmu/transfer_pool.py b/rohmu/transfer_pool.py index 41aa1b60..9d17a8f5 100644 --- a/rohmu/transfer_pool.py +++ b/rohmu/transfer_pool.py @@ -1,9 +1,6 @@ -""" -rohmu - transfer_pool - -Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/ -See LICENSE for details -""" +# Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/ +# See LICENSE for details +"""Rohmu - transfer_pool""" from __future__ import annotations diff --git a/rohmu/util.py b/rohmu/util.py index 36c3c4d2..b877639d 100644 --- a/rohmu/util.py +++ b/rohmu/util.py @@ -1,9 +1,6 @@ -""" -rohmu - common utility functions - -Copyright (c) 2022 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2022 Ohmu Ltd +# See LICENSE for details +"""Rohmu - common utility functions""" from __future__ import annotations from io import BytesIO, UnsupportedOperation diff --git a/rohmu/zstdfile.py b/rohmu/zstdfile.py index 9b06d90c..5a7045f4 100644 --- a/rohmu/zstdfile.py +++ b/rohmu/zstdfile.py @@ -1,9 +1,6 @@ -""" -rohmu - file-like interface for zstd - -Copyright (c) 2016 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2016 Ohmu Ltd +# See LICENSE for details +"""Rohmu - file-like interface for zstd""" from .common.constants import IO_BLOCK_SIZE from .filewrap import FileWrap from .typing import BinaryData, FileLike diff --git a/setup.cfg b/setup.cfg index 3d91a225..6f612d28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,9 +13,10 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Database :: Database Engines/Servers Topic :: Software Development :: Libraries :: Python Modules -url = https://github.com/aiven/rohmu/ +url = https://github.com/Aiven-Open/rohmu/ platforms = POSIX MacOS @@ -25,6 +26,7 @@ package_dir = packages = find: python_requires = >=3.8 install_requires = + azure-common >= 1.1.28, < 2 azure-storage-blob >= 2.1.0 botocore cryptography @@ -35,6 +37,7 @@ install_requires = pydantic < 2 python-dateutil python-snappy + python-swiftclient requests zstandard typing_extensions >= 3.10, < 5 diff --git a/test/common/test_statsd.py b/test/common/test_statsd.py index 7a80dd85..eeba9b21 100644 --- a/test/common/test_statsd.py +++ b/test/common/test_statsd.py @@ -1,11 +1,8 @@ -""" -Copyright (c) 2020 Aiven Ltd -See LICENSE for details - -Test that StatsClient works as advertised. +# Copyright (c) 2020 Aiven Ltd +# See LICENSE for details +"""Test that StatsClient works as advertised. This could be broader, but at least minimal functionality is tested here. - """ from __future__ import annotations diff --git a/test/notifier/test_http.py b/test/notifier/test_http.py index d42eeb00..8be5efb2 100644 --- a/test/notifier/test_http.py +++ b/test/notifier/test_http.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations from contextlib import closing, contextmanager diff --git a/test/notifier/test_interface.py b/test/notifier/test_interface.py index 23bbf69c..c2986a7d 100644 --- a/test/notifier/test_interface.py +++ b/test/notifier/test_interface.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations diff --git a/test/notifier/test_logger.py b/test/notifier/test_logger.py index b37f28c2..86c32dc5 100644 --- a/test/notifier/test_logger.py +++ b/test/notifier/test_logger.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from pytest import LogCaptureFixture from rohmu.notifier.logger import LoggerNotifier diff --git a/test/notifier/test_null.py b/test/notifier/test_null.py index 39fe546e..8a17633c 100644 --- a/test/notifier/test_null.py +++ b/test/notifier/test_null.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from rohmu.notifier.null import NullNotifier diff --git a/test/object_storage/test_azure.py b/test/object_storage/test_azure.py index 391d7c63..d05bcb67 100644 --- a/test/object_storage/test_azure.py +++ b/test/object_storage/test_azure.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from datetime import datetime from io import BytesIO from rohmu.errors import InvalidByteRangeError diff --git a/test/object_storage/test_google.py b/test/object_storage/test_google.py index 1bde75a0..67615c94 100644 --- a/test/object_storage/test_google.py +++ b/test/object_storage/test_google.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations from contextlib import ExitStack diff --git a/test/object_storage/test_local.py b/test/object_storage/test_local.py index c7e0e6e0..e844e6a5 100644 --- a/test/object_storage/test_local.py +++ b/test/object_storage/test_local.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from concurrent.futures import ThreadPoolExecutor from functools import partial from io import BytesIO diff --git a/test/object_storage/test_s3.py b/test/object_storage/test_s3.py index 231f313a..0c6a920c 100644 --- a/test/object_storage/test_s3.py +++ b/test/object_storage/test_s3.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from __future__ import annotations from botocore.response import StreamingBody diff --git a/test/object_storage/test_sftp.py b/test/object_storage/test_sftp.py index 2997da14..d4d83c20 100644 --- a/test/object_storage/test_sftp.py +++ b/test/object_storage/test_sftp.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from datetime import datetime from io import BytesIO from rohmu.object_storage.sftp import SFTPTransfer diff --git a/test/object_storage/test_swift.py b/test/object_storage/test_swift.py index 2a3eceb2..91fc142f 100644 --- a/test/object_storage/test_swift.py +++ b/test/object_storage/test_swift.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2022 Aiven, Helsinki, Finland. https://aiven.io/ from datetime import datetime from io import BytesIO from tempfile import NamedTemporaryFile diff --git a/test/test_dates.py b/test/test_dates.py index ff8ae69d..d7d70f02 100644 --- a/test/test_dates.py +++ b/test/test_dates.py @@ -1,9 +1,6 @@ -""" -rohmu test case - -Copyright (c) 2017 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2017 Ohmu Ltd +# See LICENSE for details +"""Rohmu test case""" from rohmu.dates import parse_timestamp import datetime diff --git a/test/test_encryptor.py b/test/test_encryptor.py index 9f239c36..fe8722c4 100644 --- a/test/test_encryptor.py +++ b/test/test_encryptor.py @@ -1,7 +1,6 @@ -""" -Copyright (c) 2015 Ohmu Ltd -See LICENSE for details -""" +# Copyright (c) 2015 Ohmu Ltd +# See LICENSE for details + from __future__ import annotations from py.path import LocalPath # type: ignore[import] # pylint: disable=import-error diff --git a/test/test_transfer_pool.py b/test/test_transfer_pool.py index 04c33930..94d41390 100644 --- a/test/test_transfer_pool.py +++ b/test/test_transfer_pool.py @@ -1,4 +1,4 @@ -"""Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/""" +# Copyright (c) 2023 Aiven, Helsinki, Finland. https://aiven.io/ from dataclasses import dataclass from rohmu import transfer_pool from rohmu.errors import InvalidTransferError