From cfb7702eff31322e57dbed7da202bac548f525d4 Mon Sep 17 00:00:00 2001 From: miro Date: Wed, 8 Jan 2025 16:19:47 +0000 Subject: [PATCH] pkg --- .github/FUNDING.yml | 1 + .github/workflows/build_tests.yml | 25 ++ .github/workflows/conventional-label.yaml | 10 + .github/workflows/publish_stable.yml | 58 ++++ .github/workflows/release_workflow.yml | 108 ++++++ README.md | 108 +++++- setup.py | 50 +++ {z85base91 => src}/b91.c | 23 +- src/compile.sh | 33 ++ {z85base91 => src}/z85b.c | 0 {z85base91 => src}/z85p.c | 0 z85base91/__init__.py | 202 ------------ zbase/__init__.py | 311 ++++++++++++++++++ zbase/b91.py | 100 ++++++ zbase/bench.py | 227 +++++++++++++ zbase/libbase91-aarch64.so | Bin 0 -> 69872 bytes zbase/libbase91-i386.so | Bin 0 -> 14472 bytes .../libbase91.so => zbase/libbase91-x86_64.so | Bin 15240 -> 15240 bytes zbase/libz85b-aarch64.so | Bin 0 -> 69872 bytes zbase/libz85b-i386.so | Bin 0 -> 14448 bytes .../libz85b.so => zbase/libz85b-x86_64.so | Bin zbase/libz85p-aarch64.so | Bin 0 -> 69816 bytes zbase/libz85p-i386.so | Bin 0 -> 14484 bytes .../libz85p.so => zbase/libz85p-x86_64.so | Bin zbase/version.py | 6 + zbase/z85b.py | 108 ++++++ zbase/z85p.py | 88 +++++ 27 files changed, 1236 insertions(+), 222 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/build_tests.yml create mode 100644 .github/workflows/conventional-label.yaml create mode 100644 .github/workflows/publish_stable.yml create mode 100644 .github/workflows/release_workflow.yml create mode 100644 setup.py rename {z85base91 => src}/b91.c (77%) create mode 100644 src/compile.sh rename {z85base91 => src}/z85b.c (100%) rename {z85base91 => src}/z85p.c (100%) delete mode 100644 z85base91/__init__.py create mode 100644 zbase/__init__.py create mode 100644 zbase/b91.py create mode 100644 zbase/bench.py create mode 100755 zbase/libbase91-aarch64.so create mode 100755 zbase/libbase91-i386.so rename z85base91/libbase91.so => zbase/libbase91-x86_64.so (87%) create mode 100755 zbase/libz85b-aarch64.so create mode 100755 zbase/libz85b-i386.so rename z85base91/libz85b.so => zbase/libz85b-x86_64.so (100%) create mode 100755 zbase/libz85p-aarch64.so create mode 100755 zbase/libz85p-i386.so rename z85base91/libz85p.so => zbase/libz85p-x86_64.so (100%) create mode 100644 zbase/version.py create mode 100644 zbase/z85b.py create mode 100644 zbase/z85p.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..01ecd10 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +liberapay: jarbasAI diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml new file mode 100644 index 0000000..bc795f3 --- /dev/null +++ b/.github/workflows/build_tests.yml @@ -0,0 +1,25 @@ +name: Run Build Tests +on: + push: + workflow_dispatch: + +jobs: + build_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: "3.11" + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Install package + run: | + pip install . diff --git a/.github/workflows/conventional-label.yaml b/.github/workflows/conventional-label.yaml new file mode 100644 index 0000000..0a449cb --- /dev/null +++ b/.github/workflows/conventional-label.yaml @@ -0,0 +1,10 @@ +# auto add labels to PRs +on: + pull_request_target: + types: [ opened, edited ] +name: conventional-release-labels +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: bcoe/conventional-release-labels@v1 \ No newline at end of file diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml new file mode 100644 index 0000000..0e5d94e --- /dev/null +++ b/.github/workflows/publish_stable.yml @@ -0,0 +1,58 @@ +name: Stable Release +on: + push: + branches: [master] + workflow_dispatch: + +jobs: + publish_stable: + uses: TigreGotico/gh-automations/.github/workflows/publish-stable.yml@master + secrets: inherit + with: + branch: 'master' + version_file: 'zbase/version.py' + setup_py: 'setup.py' + publish_release: true + + publish_pypi: + needs: publish_stable + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: "3.11" + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} + + + sync_dev: + needs: publish_stable + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + ref: master + - name: Push master -> dev + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: dev \ No newline at end of file diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml new file mode 100644 index 0000000..af152a0 --- /dev/null +++ b/.github/workflows/release_workflow.yml @@ -0,0 +1,108 @@ +name: Release Alpha and Propose Stable + +on: + pull_request: + types: [closed] + branches: [dev] + +jobs: + publish_alpha: + if: github.event.pull_request.merged == true + uses: TigreGotico/gh-automations/.github/workflows/publish-alpha.yml@master + secrets: inherit + with: + branch: 'dev' + version_file: 'zbase/version.py' + setup_py: 'setup.py' + update_changelog: true + publish_prerelease: true + changelog_max_issues: 100 + + notify: + if: github.event.pull_request.merged == true + needs: publish_alpha + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Send message to Matrix bots channel + id: matrix-chat-message + uses: fadenb/matrix-chat-message@v0.0.6 + with: + homeserver: 'matrix.org' + token: ${{ secrets.MATRIX_TOKEN }} + channel: '!WjxEKjjINpyBRPFgxl:krbel.duckdns.org' + message: | + new ${{ github.event.repository.name }} PR merged! https://github.com/${{ github.repository }}/pull/${{ github.event.number }} + + publish_pypi: + needs: publish_alpha + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: "3.11" + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Build Distribution Packages + run: | + python setup.py sdist bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} + + + propose_release: + needs: publish_alpha + if: success() # Ensure this job only runs if the previous job succeeds + runs-on: ubuntu-latest + steps: + - name: Checkout dev branch + uses: actions/checkout@v3 + with: + ref: dev + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.11" + + - name: Get version from setup.py + id: get_version + run: | + VERSION=$(python setup.py --version) + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Create and push new branch + run: | + git checkout -b release-${{ env.VERSION }} + git push origin release-${{ env.VERSION }} + + - name: Open Pull Request from dev to master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Variables + BRANCH_NAME="release-${{ env.VERSION }}" + BASE_BRANCH="master" + HEAD_BRANCH="release-${{ env.VERSION }}" + PR_TITLE="Release ${{ env.VERSION }}" + PR_BODY="Human review requested!" + + # Create a PR using GitHub API + curl -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -d "{\"title\":\"$PR_TITLE\",\"body\":\"$PR_BODY\",\"head\":\"$HEAD_BRANCH\",\"base\":\"$BASE_BRANCH\"}" \ + https://api.github.com/repos/${{ github.repository }}/pulls + diff --git a/README.md b/README.md index 248573c..37aef2d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,98 @@ +# Zbase - Base91 and Z85 Encodings +This repository provides C and Python implementations of three encoding schemes: **Z85P**, **Base91**, and **Z85B**. -| Encoding | Avg Encoding Time (ns) | Avg Decoding Time (ns) | Avg Size Increase | Encoding Rank | Decoding Rank | Size Increase Rank | -|-------------|-------------------------|-------------------------|-------------------|---------------|---------------|--------------------| -| [pybase64](https://github.com/mayeut/pybase64) | 1131 ns | 2946 ns | 1.35x | 1 🥇 | 1 🥇 | 4 | -| **base91** | 622324 ns | 38632 ns | 1.23x | 5 | 4 | 1 🥇 | -| [base64](https://docs.python.org/3/library/base64.html) | 7113 ns | 7051 ns | 1.35x | 3 🥉 | 3 🥉 | 4 | -| [base16](https://docs.python.org/3/library/binascii.html) | 5953 ns | 5859 ns | 2.00x | 2 🥈 | 2 🥈 | 6 | -| **z85b** | 626214 ns | 871890 ns | 1.25x | 6 | 6 | 2 🥈 | -| **z85p** | 633825 ns | 775821 ns | 1.28x | 7 | 5 | 3 🥉 | -| [base32](https://docs.python.org/3/library/base64.html) | 503698 ns | 882194 ns | 1.62x | 4 | 7 | 5 | -| [z85p_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/z85p.py) | 940859 ns | 1159043 ns | 1.28x | 8 | 8 | 3 🥉 | -| [z85b_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/z85b.py) | 983796 ns | 1314734 ns | 1.25x | 9 | 9 | 2 🥈 | -| [base91_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/b91.py) | 1414374 ns | 2080957 ns | 1.23x | 10 | 10 | 1 🥇 | +The C-based shared libraries are optimized for performance, while Python implementations provide a fallback when the C +libraries are not available. + +The repository contains: + +- **Base91 encoding**: A binary-to-text encoding scheme that uses 91 printable ASCII characters. +- **Z85B encoding**: A variant of Z85 used for efficient binary-to-text encoding. +- **Z85P encoding**: Another variant of Z85, with different padding scheme. + +## Features + +- **C-based implementation** for each encoding scheme for maximum performance. +- **Pure Python fallback** for environments where the C libraries are not available. +- Easy-to-use API for encoding and decoding with detailed error handling and logging. +- Cross-platform support (Linux, macOS, Windows) via system architecture detection. + +## Benchmarks + +| Encoding | Avg Encoding Time (ns) | Avg Decoding Time (ns) | Avg Size Increase | Encoding Rank | Decoding Rank | Size Increase Rank | +|------------------------------------------------------------------------------------------------------------------------|------------------------|------------------------|-------------------|---------------|---------------|--------------------| +| [pybase64](https://github.com/mayeut/pybase64) | 1131 ns | 2946 ns | 1.35x | 1 🥇 | 1 🥇 | 4 | +| **base91** | 622324 ns | 38632 ns | 1.23x | 5 | 4 | 1 🥇 | +| [base64](https://docs.python.org/3/library/base64.html) | 7113 ns | 7051 ns | 1.35x | 3 🥉 | 3 🥉 | 4 | +| [base16](https://docs.python.org/3/library/binascii.html) | 5953 ns | 5859 ns | 2.00x | 2 🥈 | 2 🥈 | 6 | +| **z85b** | 626214 ns | 871890 ns | 1.25x | 6 | 6 | 2 🥈 | +| **z85p** | 633825 ns | 775821 ns | 1.28x | 7 | 5 | 3 🥉 | +| [base32](https://docs.python.org/3/library/base64.html) | 503698 ns | 882194 ns | 1.62x | 4 | 7 | 5 | +| [z85p_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/z85p.py) | 940859 ns | 1159043 ns | 1.28x | 8 | 8 | 3 🥉 | +| [z85b_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/z85b.py) | 983796 ns | 1314734 ns | 1.25x | 9 | 9 | 2 🥈 | +| [base91_py](https://github.com/JarbasHiveMind/hivemind-websocket-client/blob/dev/hivemind_bus_client/encodings/b91.py) | 1414374 ns | 2080957 ns | 1.23x | 10 | 10 | 1 🥇 | + +## Usage + +You can use the provided classes to encode and decode data using the supported encoding schemes. + +### Z85P Encoding + +```python +from z85p import Z85P + +# Encode data +data = b"Hello, World!" +encoded = Z85P.encode(data) +print("Encoded Z85P:", encoded) + +# Decode data +decoded = Z85P.decode(encoded) +print("Decoded Z85P:", decoded) +``` + +### Base91 Encoding + +```python +from base91 import B91 + +# Encode data +data = b"Hello, World!" +encoded = B91.encode(data) +print("Encoded Base91:", encoded) + +# Decode data +decoded = B91.decode(encoded) +print("Decoded Base91:", decoded) +``` + +### Z85B Encoding + +```python +from z85b import Z85B + +# Encode data +data = b"Hello, World!" +encoded = Z85B.encode(data) +print("Encoded Z85B:", encoded) + +# Decode data +decoded = Z85B.decode(encoded) +print("Decoded Z85B:", decoded) +``` + +## Error Handling + +The library automatically falls back to the Python implementation if the C libraries are not found or fail to load. Any +issues related to encoding or decoding will raise a `ValueError` with a detailed message. + +In the case of missing C libraries, warnings will be logged using Python's built-in `logging` module. + +### Logging + +The library uses the `logging` module to provide useful runtime information: + +```bash +2025-01-08 12:34:56,789 - WARNING - Z85P C library not available: Library load error. Falling back to pure Python implementation. +``` diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..42c9011 --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +import os +from setuptools import setup + +BASEDIR = os.path.abspath(os.path.dirname(__file__)) + + +def get_version(): + """ Find the version of the package""" + version_file = os.path.join(BASEDIR, 'zbase', 'version.py') + major, minor, build, alpha = (None, None, None, None) + with open(version_file) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if int(alpha): + version += f"a{alpha}" + return version + + +def required(requirements_file): + """ Read requirements file and remove comments and empty lines. """ + with open(os.path.join(BASEDIR, requirements_file), 'r') as f: + requirements = f.read().splitlines() + return [pkg for pkg in requirements + if pkg.strip() and not pkg.startswith("#")] + + + +setup( + name='zbase', + version=get_version(), + packages=['zbase'], + url='https://github.com/JarbasHiveMind/zbase', + license='Apache-2.0', + author='jarbasAi', + include_package_data=True, + author_email='jarbasai@mailfence.com', + description='base91, z85b and z85p encodings' +) diff --git a/z85base91/b91.c b/src/b91.c similarity index 77% rename from z85base91/b91.c rename to src/b91.c index a4c0141..19b7091 100644 --- a/z85base91/b91.c +++ b/src/b91.c @@ -6,13 +6,19 @@ #define ALPHABET_SIZE 91 // Base91 Alphabet (same as Python ALPHABET) -const char ALPHABET[ALPHABET_SIZE] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,-./:;<=>?@[\\]^_`{|}~\""; +const char ALPHABET[ALPHABET_SIZE] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,./:;<=>?@[]^_`{|}~\""; // Decode table (maps Base91 characters to their indices in the alphabet) int DECODE_TABLE[256]; // Function to initialize the DECODE_TABLE void initialize_decode_table() { + // Set all values to -1 (invalid by default) + for (int i = 0; i < 256; i++) { + DECODE_TABLE[i] = -1; + } + + // Now populate the DECODE_TABLE for valid characters for (int i = 0; i < ALPHABET_SIZE; i++) { DECODE_TABLE[(unsigned char)ALPHABET[i]] = i; } @@ -72,15 +78,15 @@ char* encode(const unsigned char* data, size_t data_len, size_t* output_len) { size_t out_idx = 0; for (size_t i = 0; i < data_len; i++) { - b |= data[i] << n; - n += 8; - if (n > 13) { - int v = b & 8191; + b |= data[i] << n; // Shift the bits + n += 8; // Move 8 bits at a time + if (n > 13) { // We need to encode multiple characters + int v = b & 8191; // Get the first 13 bits (Base91 split) if (v > 88) { b >>= 13; n -= 13; } else { - v = b & 16383; + v = b & 16383; // Get 14 bits b >>= 14; n -= 14; } @@ -89,14 +95,15 @@ char* encode(const unsigned char* data, size_t data_len, size_t* output_len) { } } + // Handle any leftover bits if (n) { - out[out_idx++] = ALPHABET[b % 91]; + out[out_idx++] = ALPHABET[b % 91]; // Place the final character if (n > 7 || b > 90) { out[out_idx++] = ALPHABET[b / 91]; } } - out[out_idx] = '\0'; + out[out_idx] = '\0'; // Null-terminate the output string *output_len = out_idx; return out; } diff --git a/src/compile.sh b/src/compile.sh new file mode 100644 index 0000000..1f9a249 --- /dev/null +++ b/src/compile.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Compile for x86_64 +gcc -shared -o libbase91-x86_64.so -fPIC b91.c +gcc -shared -o libz85p-x86_64.so -fPIC z85p.c +gcc -shared -o libz85b-x86_64.so -fPIC z85b.c + +# Compile for armhf (32-bit ARM) +arm-linux-gnueabihf-gcc -shared -o libbase91-armhf.so -fPIC b91.c +arm-linux-gnueabihf-gcc -shared -o libz85p-armhf.so -fPIC z85p.c +arm-linux-gnueabihf-gcc -shared -o libz85b-armhf.so -fPIC z85b.c + +# Compile for aarch64 (64-bit ARM) +aarch64-linux-gnu-gcc -shared -o libbase91-aarch64.so -fPIC b91.c +aarch64-linux-gnu-gcc -shared -o libz85p-aarch64.so -fPIC z85p.c +aarch64-linux-gnu-gcc -shared -o libz85b-aarch64.so -fPIC z85b.c + +# Compile for i386 (32-bit Intel/AMD) +gcc -m32 -shared -o libbase91-i386.so -fPIC b91.c +gcc -m32 -shared -o libz85p-i386.so -fPIC z85p.c +gcc -m32 -shared -o libz85b-i386.so -fPIC z85b.c + +# Compile for Windows +x86_64-w64-mingw32-gcc -shared -o libbase91.dll -fPIC b91.c +x86_64-w64-mingw32-gcc -shared -o libz85p.dll -fPIC z85p.c +x86_64-w64-mingw32-gcc -shared -o libz85b.dll -fPIC z85b.c + +# Compile for MacOS +osxcross -shared -o libbase91.dylib -fPIC b91.c +osxcross -shared -o libz85p.dylib -fPIC z85p.c +osxcross -shared -o libz85b.dylib -fPIC z85b.c + +echo "Compilation completed!" diff --git a/z85base91/z85b.c b/src/z85b.c similarity index 100% rename from z85base91/z85b.c rename to src/z85b.c diff --git a/z85base91/z85p.c b/src/z85p.c similarity index 100% rename from z85base91/z85p.c rename to src/z85p.c diff --git a/z85base91/__init__.py b/z85base91/__init__.py deleted file mode 100644 index a8240ba..0000000 --- a/z85base91/__init__.py +++ /dev/null @@ -1,202 +0,0 @@ -# TODO - package this properly for pypi -# $ gcc -shared -o libz85p.so -fPIC z85p.c -# $ gcc -shared -o libz85b.so -fPIC z85b.c -# $ gcc -shared -o libbase91.so -fPIC b91.c - -import ctypes -from ctypes import c_char_p, c_void_p, c_size_t, c_ubyte, byref, POINTER -from typing import Union - - -class Z85P: - # Load the shared library - lib = ctypes.CDLL('./libz85p.so') # On Windows, use './z85p.dll' - - # Initialize the Z85 map (this needs to be called first) - lib.initialize_z85_map() - - # Define the encode function prototype - lib.encode_z85p.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, POINTER(c_size_t)] - lib.encode_z85p.restype = ctypes.POINTER(ctypes.c_ubyte) - - # Define the decode function prototype - lib.decode_z85p.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, POINTER(c_size_t)] - lib.decode_z85p.restype = ctypes.POINTER(ctypes.c_ubyte) - - @classmethod - def encode(cls, data: bytes) -> bytes: - out_len = c_size_t(0) - raw_data = (ctypes.c_ubyte * len(data))(*data) - encoded_data = cls.lib.encode_z85p(raw_data, len(data), ctypes.byref(out_len)) - return bytes(ctypes.string_at(encoded_data, out_len.value)) - - @classmethod - def decode(cls, data: bytes) -> bytes: - out_len = c_size_t(0) - raw_data = (ctypes.c_ubyte * len(data))(*data) - decoded_data = cls.lib.decode_z85p(raw_data, len(data), ctypes.byref(out_len)) - return bytes(ctypes.string_at(decoded_data, out_len.value)) - - -class B91: - # Load the shared library - lib = ctypes.CDLL('./libbase91.so') # On Windows, use './base91.dll' - - # Initialize the decode table (this needs to be called first) - lib.initialize_decode_table() - - # Define the decode function prototype - lib.decode.argtypes = [c_char_p, ctypes.POINTER(c_size_t)] - lib.decode.restype = c_void_p - - # Define the encode function prototype - lib.encode.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, ctypes.POINTER(c_size_t)] - lib.encode.restype = c_char_p - - @classmethod - def decode(cls, encoded_data: Union[str, bytes]): - if isinstance(encoded_data, str): - # Convert the encoded data to bytes - encoded_data = encoded_data.encode('utf-8') - output_len = c_size_t(0) - - # Call the C function - decoded_data = cls.lib.decode(encoded_data, ctypes.byref(output_len)) - - if decoded_data: - return ctypes.string_at(decoded_data, output_len.value) - else: - raise ValueError("Invalid Base91 string") - - @classmethod - def encode(cls, data: Union[str, bytes]): - if isinstance(data, str): - # Convert the data to bytes - data = data.encode('utf-8') - output_len = c_size_t(0) - - # Call the C function - encoded_data = cls.lib.encode((ctypes.c_ubyte * len(data))(*data), len(data), ctypes.byref(output_len)) - - if encoded_data: - return ctypes.string_at(encoded_data, output_len.value) - else: - raise ValueError("Encoding failed") - - -class Z85B: - # Load the shared library dynamically - lib = ctypes.CDLL('./libz85b.so') # Update path as needed - - # Define function prototypes - lib.encode_z85b.argtypes = [POINTER(c_ubyte), c_size_t, POINTER(c_size_t)] - lib.encode_z85b.restype = POINTER(c_ubyte) - - lib.decode_z85b.argtypes = [POINTER(c_ubyte), c_size_t, POINTER(c_size_t)] - lib.decode_z85b.restype = POINTER(c_ubyte) - - lib.free.argtypes = [ctypes.c_void_p] # Add free function for memory cleanup - - @classmethod - def encode(cls, data: bytes) -> bytes: - """ - Encode raw bytes into Z85b format. - - Args: - data (bytes): Input data to encode. - - Returns: - bytes: Z85b-encoded data. - - Raises: - ValueError: If encoding fails. - """ - output_len = c_size_t(0) - encoded_data = cls.lib.encode_z85b((c_ubyte * len(data))(*data), len(data), byref(output_len)) - if not encoded_data: - raise ValueError("Encoding failed") - - try: - return ctypes.string_at(encoded_data, output_len.value) - finally: - cls.lib.free(encoded_data) - - @classmethod - def decode(cls, encoded_data: bytes) -> bytes: - """ - Decode Z85b-encoded bytes into raw bytes. - - Args: - encoded_data (bytes): Z85b-encoded input. - - Returns: - bytes: Decoded raw bytes. - - Raises: - ValueError: If decoding fails. - """ - output_len = c_size_t(0) - decoded_data = cls.lib.decode_z85b((c_ubyte * len(encoded_data))(*encoded_data), len(encoded_data), - byref(output_len)) - if not decoded_data: - raise ValueError("Decoding failed") - - try: - return ctypes.string_at(decoded_data, output_len.value) - finally: - cls.lib.free(decoded_data) - - -if __name__ == "__main__": - from hivemind_bus_client.encodings import Z85B as Z85Bpy, B91 as B91py, Z85P as Z85Ppy - - - def test_b91(s=b"Hello, Base91!"): - # Example usage: - try: - encoded = B91py.encode(s) - print("Encoded py:", encoded) - decoded = B91py.decode(encoded) - print("Decoded py:", decoded) - - encoded = B91.encode(s) - print("Encoded:", encoded) - decoded = B91.decode(encoded) - print("Decoded:", decoded) - except Exception as e: - print(f"Error: {e}") - - - def test_z85b(s=b"Hello, Z85B!"): - try: - encoded = Z85Bpy.encode(s) - print("Encoded py:", encoded) - decoded = Z85Bpy.decode(encoded) - print("Decoded py:", decoded) - - encoded = Z85B.encode(s) - print("Encoded:", encoded) - decoded = Z85B.decode(encoded) - print("Decoded:", decoded) - except Exception as e: - print(f"Error: {e}") - - - def test_z85p(s=b"Hello, Z85P!"): - try: - encoded = Z85Ppy.encode(s) - print(f"Encoded py: {encoded}") - decoded = Z85Ppy.decode(encoded) - print(f"Decoded py: {decoded.decode('utf-8')}") - - encoded = Z85P.encode(s) - print(f"Encoded: {encoded}") - decoded = Z85P.decode(encoded) - print(f"Decoded: {decoded.decode('utf-8')}") - except Exception as e: - print(f"Error: {e}") - - - test_b91() - test_z85p() - test_z85b() diff --git a/zbase/__init__.py b/zbase/__init__.py new file mode 100644 index 0000000..f0a022e --- /dev/null +++ b/zbase/__init__.py @@ -0,0 +1,311 @@ +import ctypes +import logging +import os.path +import platform +from ctypes import c_char_p, c_void_p, c_size_t, c_ubyte, byref, POINTER +from typing import Union + +# Set up logging +logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s') + + +# get the appropriate shared library file based on architecture +def get_arch_lib(lib_base_name: str) -> str: + """ + Returns the path to the shared library based on the system's architecture. + + Args: + lib_base_name (str): The base name of the shared library (e.g., 'libz85p'). + + Returns: + str: The path to the shared library for the appropriate system architecture. + + Raises: + ValueError: If the system architecture is unsupported. + """ + arch = platform.machine() # Get system architecture + if arch == "x86_64": + return f"{os.path.dirname(__file__)}/{lib_base_name}-x86_64.so" + elif arch == "aarch64": + return f"{os.path.dirname(__file__)}/{lib_base_name}-aarch64.so" + elif arch == "i386" or arch == "i686": + return f"{os.path.dirname(__file__)}/{lib_base_name}-i386.so" + else: + raise ValueError(f"Unsupported architecture: {arch}") + + +try: + class Z85P: + """ + Class for encoding and decoding Z85P format using a C-based shared library. + If the C library is not available, it falls back to a pure Python implementation. + """ + # Load the correct shared library based on system architecture + lib = ctypes.CDLL(get_arch_lib('libz85p')) + + # Initialize the Z85 map (this needs to be called first) + lib.initialize_z85_map() + + # Define the encode function prototype + lib.encode_z85p.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, POINTER(c_size_t)] + lib.encode_z85p.restype = ctypes.POINTER(ctypes.c_ubyte) + + # Define the decode function prototype + lib.decode_z85p.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, POINTER(c_size_t)] + lib.decode_z85p.restype = ctypes.POINTER(ctypes.c_ubyte) + + @classmethod + def encode(cls, data: bytes) -> bytes: + """ + Encodes the input data into Z85P format. + + Args: + data (bytes): The raw data to encode. + + Returns: + bytes: The Z85P-encoded data. + + Raises: + ValueError: If encoding fails. + """ + out_len = c_size_t(0) + raw_data = (ctypes.c_ubyte * len(data))(*data) + encoded_data = cls.lib.encode_z85p(raw_data, len(data), ctypes.byref(out_len)) + if not encoded_data: + raise ValueError("Encoding failed") + return bytes(ctypes.string_at(encoded_data, out_len.value)) + + @classmethod + def decode(cls, data: bytes) -> bytes: + """ + Decodes the input Z85P-encoded data into raw bytes. + + Args: + data (bytes): The Z85P-encoded data to decode. + + Returns: + bytes: The decoded raw data. + + Raises: + ValueError: If decoding fails. + """ + out_len = c_size_t(0) + raw_data = (ctypes.c_ubyte * len(data))(*data) + decoded_data = cls.lib.decode_z85p(raw_data, len(data), ctypes.byref(out_len)) + if not decoded_data: + raise ValueError("Decoding failed") + return bytes(ctypes.string_at(decoded_data, out_len.value)) +except Exception as e: + logging.warning(f"Z85P C library not available: {e}. Falling back to pure Python implementation.") + from .z85p import Z85P + +try: + class B91: + """ + Class for encoding and decoding Base91 format using a C-based shared library. + If the C library is not available, it falls back to a pure Python implementation. + """ + # Load the correct shared library based on system architecture + lib = ctypes.CDLL(get_arch_lib('libbase91')) + + # Initialize the decode table (this needs to be called first) + lib.initialize_decode_table() + + # Define the decode function prototype + lib.decode.argtypes = [c_char_p, ctypes.POINTER(c_size_t)] + lib.decode.restype = c_void_p + + # Define the encode function prototype + lib.encode.argtypes = [ctypes.POINTER(ctypes.c_ubyte), c_size_t, ctypes.POINTER(c_size_t)] + lib.encode.restype = c_char_p + + @classmethod + def decode(cls, encoded_data: Union[str, bytes]) -> bytes: + """ + Decodes the input Base91-encoded data into raw bytes. + + Args: + encoded_data (Union[str, bytes]): The Base91-encoded data to decode. + + Returns: + bytes: The decoded raw data. + + Raises: + ValueError: If decoding fails. + """ + if isinstance(encoded_data, str): + # Convert the encoded data to bytes + encoded_data = encoded_data.encode('utf-8') + output_len = c_size_t(0) + + # Call the C function + decoded_data = cls.lib.decode(encoded_data, ctypes.byref(output_len)) + + if not decoded_data: + raise ValueError("Invalid Base91 string") + return ctypes.string_at(decoded_data, output_len.value) + + @classmethod + def encode(cls, data: Union[str, bytes]) -> bytes: + """ + Encodes the input data into Base91 format. + + Args: + data (Union[str, bytes]): The raw data to encode. + + Returns: + bytes: The Base91-encoded data. + + Raises: + ValueError: If encoding fails. + """ + if isinstance(data, str): + # Convert the data to bytes + data = data.encode('utf-8') + output_len = c_size_t(0) + + # Call the C function + encoded_data = cls.lib.encode((ctypes.c_ubyte * len(data))(*data), len(data), ctypes.byref(output_len)) + + if not encoded_data: + raise ValueError("Encoding failed") + return ctypes.string_at(encoded_data, output_len.value) +except Exception as e: + logging.warning(f"Base91 C library not available: {e}. Falling back to pure Python implementation.") + from .b91 import B91 + +try: + class Z85B: + """ + Class for encoding and decoding Z85B format using a C-based shared library. + If the C library is not available, it falls back to a pure Python implementation. + """ + # Load the correct shared library based on system architecture + lib = ctypes.CDLL(get_arch_lib('libz85b')) + + # Define function prototypes + lib.encode_z85b.argtypes = [POINTER(c_ubyte), c_size_t, POINTER(c_size_t)] + lib.encode_z85b.restype = POINTER(c_ubyte) + + lib.decode_z85b.argtypes = [POINTER(c_ubyte), c_size_t, POINTER(c_size_t)] + lib.decode_z85b.restype = POINTER(c_ubyte) + + lib.free.argtypes = [ctypes.c_void_p] # Add free function for memory cleanup + + @classmethod + def encode(cls, data: bytes) -> bytes: + """ + Encodes the input data into Z85B format. + + Args: + data (bytes): The raw data to encode. + + Returns: + bytes: The Z85B-encoded data. + + Raises: + ValueError: If encoding fails. + """ + output_len = c_size_t(0) + encoded_data = cls.lib.encode_z85b((c_ubyte * len(data))(*data), len(data), byref(output_len)) + if not encoded_data: + raise ValueError("Encoding failed") + + try: + return ctypes.string_at(encoded_data, output_len.value) + finally: + cls.lib.free(encoded_data) + + @classmethod + def decode(cls, encoded_data: bytes) -> bytes: + """ + Decodes the input Z85B-encoded data into raw bytes. + + Args: + encoded_data (bytes): The Z85B-encoded data to decode. + + Returns: + bytes: The decoded raw data. + + Raises: + ValueError: If decoding fails. + """ + output_len = c_size_t(0) + decoded_data = cls.lib.decode_z85b((c_ubyte * len(encoded_data))(*encoded_data), len(encoded_data), + byref(output_len)) + if not decoded_data: + raise ValueError("Decoding failed") + + try: + return ctypes.string_at(decoded_data, output_len.value) + finally: + cls.lib.free(decoded_data) +except Exception as e: + logging.warning(f"Z85B C library not available: {e}. Falling back to pure Python implementation.") + from .z85b import Z85B + +if __name__ == "__main__": + + from zbase.b91 import B91 as B91py + from zbase.z85b import Z85B as Z85Bpy + from zbase.z85p import Z85P as Z85Ppy + + + def test_b91(s=b"Hello, Base91!"): + # Example usage: + try: + pencoded = B91py.encode(s) + print("Encoded py:", pencoded) + pdecoded = B91py.decode(pencoded) + print("Decoded py:", pdecoded) + + encoded = B91.encode(s) + print("Encoded:", encoded) + decoded = B91.decode(encoded) + print("Decoded:", decoded) + + assert pdecoded == decoded + assert pencoded == encoded + except Exception as e: + print(f"Error: {e}") + + + def test_z85b(s=b"Hello, Z85B!"): + try: + pencoded = Z85Bpy.encode(s) + print("Encoded py:", pencoded) + pdecoded = Z85Bpy.decode(pencoded) + print("Decoded py:", pdecoded) + + encoded = Z85B.encode(s) + print("Encoded:", encoded) + decoded = Z85B.decode(encoded) + print("Decoded:", decoded) + + assert pdecoded == decoded + assert pencoded == encoded + except Exception as e: + print(f"Error: {e}") + + + def test_z85p(s=b"Hello, Z85P!"): + try: + pencoded = Z85Ppy.encode(s) + print(f"Encoded py: {pencoded}") + pdecoded = Z85Ppy.decode(pencoded) + print(f"Decoded py: {pdecoded.decode('utf-8')}") + + encoded = Z85P.encode(s) + print(f"Encoded: {encoded}") + decoded = Z85P.decode(encoded) + print(f"Decoded: {decoded.decode('utf-8')}") + + assert pdecoded == decoded + assert pencoded == encoded + except Exception as e: + print(f"Error: {e}") + + + test_b91() + test_z85p() + test_z85b() diff --git a/zbase/b91.py b/zbase/b91.py new file mode 100644 index 0000000..6ce0773 --- /dev/null +++ b/zbase/b91.py @@ -0,0 +1,100 @@ +from typing import Union + + +class B91: + ALPHABET = [ + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '#', '$', + '%', '&', '(', ')', '*', '+', ',', '.', '/', ':', ';', '<', '=', + '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~', '"' + ] + + DECODE_TABLE = {char: idx for idx, char in enumerate(ALPHABET)} + + @classmethod + def decode(cls, encoded_data: Union[str, bytes], encoding: str = "utf-8") -> bytes: + """ + Decodes a Base91-encoded string into its original binary form. + + Args: + encoded_data (Union[str, bytes]): Base91-encoded input data. If `bytes`, it is decoded as UTF-8. + encoding (str): The encoding to use if `encoded_data` is provided as a string. Default is 'utf-8'. + + Returns: + bytes: The decoded binary data. + + Raises: + ValueError: If the input contains invalid Base91 characters. + """ + if isinstance(encoded_data, bytes): + encoded_data = encoded_data.decode(encoding) + + v = -1 + b = 0 + n = 0 + out = bytearray() + + for char in encoded_data: + if char not in cls.DECODE_TABLE: + raise ValueError(f"Invalid Base91 character: {char}") + c = cls.DECODE_TABLE[char] + if v < 0: + v = c + else: + v += c * 91 + b |= v << n + n += 13 if (v & 8191) > 88 else 14 + while n >= 8: + out.append(b & 255) + b >>= 8 + n -= 8 + v = -1 + + if v >= 0: + out.append((b | v << n) & 255) + + return bytes(out) + + @classmethod + def encode(cls, data: Union[bytes, str], encoding: str = "utf-8") -> bytes: + """ + Encodes binary data into a Base91-encoded string. + + Args: + data (Union[bytes, str]): Input binary data to encode. If `str`, it is encoded as UTF-8. + encoding (str): The encoding to use if `data` is provided as a string. Default is 'utf-8'. + + Returns: + str: The Base91-encoded string. + """ + if isinstance(data, str): + data = data.encode(encoding) + + b = 0 + n = 0 + out = [] + + for byte in data: + b |= byte << n + n += 8 + if n > 13: + v = b & 8191 + if v > 88: + b >>= 13 + n -= 13 + else: + v = b & 16383 + b >>= 14 + n -= 14 + out.append(cls.ALPHABET[v % 91]) + out.append(cls.ALPHABET[v // 91]) + + if n: + out.append(cls.ALPHABET[b % 91]) + if n > 7 or b > 90: + out.append(cls.ALPHABET[b // 91]) + + return ''.join(out).encode(encoding) diff --git a/zbase/bench.py b/zbase/bench.py new file mode 100644 index 0000000..7cd398d --- /dev/null +++ b/zbase/bench.py @@ -0,0 +1,227 @@ +import base64 +import pybase64 +import random +import string +import time +from typing import Callable, Dict, List, Tuple + +import click +from z85base91 import Z85P, Z85B, B91 +from hivemind_bus_client.encodings import Z85B as Z85Bpy, Z85P as Z85Ppy, B91 as B91py +from tabulate import tabulate + + +def get_encoder(encoding: str) -> Callable[[bytes], bytes]: + """Retrieve the encoder function for the given encoding.""" + encoders = { + "base64": base64.b64encode, + "base64_py": pybase64.b64encode, + "z85b": Z85B.encode, + "z85p": Z85P.encode, + "base91": B91.encode, + "z85b_py": Z85Bpy.encode, + "z85p_py": Z85Ppy.encode, + "base91_py": B91py.encode, + "base32": base64.b32encode + } + return encoders[encoding] + + +def get_decoder(encoding: str) -> Callable[[bytes], bytes]: + """Retrieve the decoder function for the given encoding.""" + decoders = { + "base64": base64.b64decode, + "base64_py": pybase64.b64decode, + "z85b": Z85B.decode, + "z85p": Z85P.decode, + "base91": B91.decode, + "z85b_py": Z85Bpy.decode, + "z85p_py": Z85Ppy.decode, + "base91_py": B91py.decode, + "base32": base64.b32decode + } + return decoders[encoding] + + +def generate_random_data(size: int) -> bytes: + """Generate random binary data of a given size.""" + return ''.join(random.choices(string.ascii_letters + string.digits, k=size)).encode("utf-8") + + +def benchmark_encoding(encoding: str, data: bytes) -> Dict[str, int]: + """Benchmark encoding and decoding for a given encoding.""" + encoder = get_encoder(encoding) + decoder = get_decoder(encoding) + + # Measure encoding time in nanoseconds + start_time = time.perf_counter_ns() + encoded_data = encoder(data) + encoding_time = time.perf_counter_ns() - start_time + + # Measure decoding time in nanoseconds + start_time = time.perf_counter_ns() + decoded_data = decoder(encoded_data) + decoding_time = time.perf_counter_ns() - start_time + + # Validate decoding + if decoded_data != data: + raise ValueError(f"Decoded data does not match for encoding {encoding}.") + + # Calculate size increase + original_size = len(data) + encoded_size = len(encoded_data) + size_increase = encoded_size / original_size + + return { + "encoding_time": encoding_time, + "decoding_time": decoding_time, + "size_increase": size_increase, + } + + +def get_rankings(metric: Dict[str, Dict[str, int]], key: str) -> List[Tuple[str, int]]: + """Rank the encodings based on the provided metric, handling ties.""" + sorted_encodings = sorted(metric.items(), key=lambda x: x[1][key], reverse=False) + rankings = [] + current_rank = 1 # Start from rank 1 + + for i in range(len(sorted_encodings)): + if i > 0 and sorted_encodings[i][1][key] == sorted_encodings[i - 1][1][key]: + # Tie case: Same rank as the previous item + rankings.append((sorted_encodings[i][0], rankings[-1][1])) + else: + # No tie, increase the rank + rankings.append((sorted_encodings[i][0], current_rank)) + current_rank += 1 # Increment rank only when no tie + + return rankings + + +def compare_python_c(encoding: str, python_results: Dict[str, Dict[str, int]], + c_results: Dict[str, Dict[str, int]]) -> float: + """Compare the speed between Python and C encodings and calculate how many times faster or slower it is.""" + if encoding.endswith("_py"): + c_encoding = encoding[:-3] # Remove the _py suffix to get the C counterpart + python_time = python_results[encoding]["encoding_time"] + c_time = c_results[c_encoding]["encoding_time"] + + if c_time == 0: + return float('inf') # Avoid division by zero + return python_time / c_time + return 1.0 # If not a Python version, return 1 (no comparison) + + +@click.command() +@click.option("--sizes", default="100,1000,10000", help="Comma-separated list of data sizes to test.") +@click.option("--iterations", default=10, help="Number of iterations for each test.") +def main(sizes: str, iterations: int): + sizes = list(map(int, sizes.split(","))) + + encodings = [ + "base64", + "base32", + "base64_py", + "z85b", + "z85p", + "base91", + "z85b_py", + "z85p_py", + "base91_py", + ] + + results = {size: {encoding: [] for encoding in encodings} for size in sizes} + + # Run benchmarks + for size in sizes: + print(f"Testing size: {size} bytes") + for _ in range(iterations): + data = generate_random_data(size) + for encoding in encodings: + result = benchmark_encoding(encoding, data) + results[size][encoding].append(result) + + # Calculate averages and print results + global_ranking = {encoding: {"encoding_time": 0, "decoding_time": 0, "size_increase": 0} for encoding in encodings} + table = [] + + # Aggregate results across all sizes + for encoding in encodings: + total_encoding_time = 0 + total_decoding_time = 0 + total_size_increase = 0 + + for size, encoding_results in results.items(): + avg_encoding_time = sum(m["encoding_time"] for m in encoding_results[encoding]) // iterations + avg_decoding_time = sum(m["decoding_time"] for m in encoding_results[encoding]) // iterations + avg_size_increase = sum(m["size_increase"] for m in encoding_results[encoding]) / iterations + + total_encoding_time += avg_encoding_time + total_decoding_time += avg_decoding_time + total_size_increase += avg_size_increase + + table.append([ + encoding, + f"{avg_encoding_time} ns", + f"{avg_decoding_time} ns", + f"{avg_size_increase:.2f}x size increase" + ]) + + # Store global averages + global_ranking[encoding]["encoding_time"] = total_encoding_time // len(sizes) + global_ranking[encoding]["decoding_time"] = total_decoding_time // len(sizes) + global_ranking[encoding]["size_increase"] = total_size_increase / len(sizes) + + # Global ranking (based on average times) + print("\n### Global Ranking (Merged) ###") + + # Get rankings for encoding time, decoding time, and size increase + sorted_by_encoding_time = get_rankings(global_ranking, "encoding_time") + sorted_by_decoding_time = get_rankings(global_ranking, "decoding_time") + sorted_by_size_increase = get_rankings(global_ranking, "size_increase") + + merged_table = [] + for encoding, metrics in global_ranking.items(): + encoding_time_rank = next(rank for enc, rank in sorted_by_encoding_time if enc == encoding) + decoding_time_rank = next(rank for enc, rank in sorted_by_decoding_time if enc == encoding) + size_increase_rank = next(rank for enc, rank in sorted_by_size_increase if enc == encoding) + + # Calculate the average rank + avg_rank = (encoding_time_rank + decoding_time_rank + size_increase_rank) / 3 + + # Medal assignments + encoding_time_medal = "🥇" if encoding_time_rank == 1 else "🥈" if encoding_time_rank == 2 else "🥉" if encoding_time_rank == 3 else "" + decoding_time_medal = "🥇" if decoding_time_rank == 1 else "🥈" if decoding_time_rank == 2 else "🥉" if decoding_time_rank == 3 else "" + size_increase_medal = "🥇" if size_increase_rank == 1 else "🥈" if size_increase_rank == 2 else "🥉" if size_increase_rank == 3 else "" + + # Add the top-ranked emojis + merged_table.append([ + encoding, + f"{metrics['encoding_time']} ns", + f"{metrics['decoding_time']} ns", + f"{metrics['size_increase']:.2f}x", + f"{encoding_time_rank} {encoding_time_medal}", + f"{decoding_time_rank} {decoding_time_medal}", + f"{size_increase_rank} {size_increase_medal}", + avg_rank + ]) + + # Pairwise comparison for Python vs C + if encoding.endswith("_py"): # If it's a Python version + speed_comparison = compare_python_c(encoding, global_ranking, global_ranking) + if float(speed_comparison) > 1: + merged_table[-1].append(f"{speed_comparison:.2f}x slower") + else: + merged_table[-1].append(f"{1/speed_comparison:.2f}x faster") + + # Sort the merged table based on the average rank + merged_table.sort(key=lambda x: float(str(x[-2]).split()[0]), reverse=False) + + # Display the final table + print(tabulate(merged_table, + headers=["Encoding", "Avg Encoding Time (ns)", "Avg Decoding Time (ns)", "Avg Size Increase", + "Encoding Rank", "Decoding Rank", "Size Increase Rank", "Score", "Reference vs Optimized"], + tablefmt="grid")) + + +if __name__ == "__main__": + main() diff --git a/zbase/libbase91-aarch64.so b/zbase/libbase91-aarch64.so new file mode 100755 index 0000000000000000000000000000000000000000..8a857675732da5cef769b313d3b2d27d60bb986e GIT binary patch literal 69872 zcmeI0e{5S<6~~Y5uD_hNX`20ND&n+-m2NZbD6HXE?fj_IEor;pZi50ZPV6*hj@{bs zmabt7gFng!qG5<3p^Z(d#E(DFK>Sz`gRXy|34}CgnToW5Qx#N+fif}B49(2<+;>m% z{G($a{yeXGIq#l(&bjA)?z!(>_rc!&-F}~s5w#qSRIEu?O3-b^&x&OtlgmAHf~7m;>4 zW^S#B5x^bWdw`Cfn!Nbec=vt3-v9gy>wkQE^SzxNF&={ZSK?+r&dF0T z4>i#5hHhg6{WNrq;_$pe4eY;q1^b(>VE+l|*U~kpbeq3j*aJqtil!`)gJ|>hIPMk- z(!~{~Nhhq^XRQ9*GFRVh_4inPkJX=ayNMj<(0DHE6pD#_(Q(Mx6CZGrseI~Cx=>8z z;{)B9Ts9R?3};eqU*jHUWFp~=rn8Am`u-F}`}g-nyL#i))7!nTr`L%`yZU=6olO_h z#sCA1vbeg8|2|C#Uxr^w}r-|aT@auRfR!+cgF5vq%Ay4+LweKMM<;Keo zo{ctr`jL*y(MO+y{RQ}T&Be;Y#@D~#JAeGAhR5H6R9=VihR#GfBR;y-npKxDL6F~8XMX<8=$Qcr(-hm z=!IZp#>9a%ao|_NbNiU@$jyi&`5=#JW;B2N$7Y@wuRM#nV$AZxm`kah%7mOs4$V{! z;=YS!X4@m>67zli!L!&kkI-umxbkxMS&TJZYJ-1Ur2G*3UyJ8!BW2Xnt5}yfV*Tb2 z@~_-L<;o^1-^O#nSfx#oa){^=qDW`y>(n{>bx5uQaN+sTMB zoH-6{qn7p|-;r^|deS#Fh@IGa8gdps#vfePKv%jUQpS5^dKUAWMV>q-j@7Y&T6{#E zB|OvQ@^vOId?`eCl;F!_w{M^$O;{V@%}fkWa6-=I61a1%f94;RxXdl+$J{)5`!*0_ zBQ7My-sds0+?#1+TGP zZ|@r8+tgZ-nz<2c4ITr`9!xkN=E7@j>XunEb9XJanzi6Lgi!y?AJ6SO=*Yk_%+;<% zDF%HEdaTJRpVd#UTDRc416<*Y_j9Z9*+DJFhy5p4FWT*N?KoBfHQ{f1ZavHBq^~VN zzS~PT+V#X7W>M3ew|Q4!?Y5XSiBJD9@0;xSTF2#~v-p+;J0_Q| zqsjJln~2_eEbuXmxrXq}+H)2d5#*8BWVa%mt(%*($%=q|r zOkcoUE?_PZ&s=zqEzE_-KrMcr_YHXQJL3ynXLxQj-v2U3X31=sC+5@1@0(d>&2@-A zsY8BG$u~*9N71hCp5EQDzCE|y-aoK+-~KxWKO8@B=UsPy5^ zkAK1$IzIU+!m7U?T!_Dyr^=Pe-H_)XGmy>ytW>5TlaQw%#~^8Q9<`uyC<{5;y0K{l~t|!|>RE$1mgFfIQ7vGP-)`@unw&{)c}5 zG6&@T@8fQ|T&b8dsPsSS_M=NLC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwN!4tM$uC>ZDq~ zf@FVG>sOL+tJVi;NDAB25uhfLedSSOwyS7LHln8^K&$DchWa%Wl8str-%PSkJ!-^X zOR_(!^)1w>o&vOvp0@j|+P-yR-&O0cCaI5V{WY}B)=jlucIBd$%VP<>T3OV%PwxJ# z=gIE{KP~EI-@Eqp>*ea}`SU)%^0+JLlDx2NViISfW?kCfG)|DFYU*LD&A(QElhr?6 zW53IcU(dPA+ROXhnBmXexP7$9U$&ncr@kJN#=c(v2=vTZz9VRw%Z*cCZ;zqZNA-Hw zehuxYsasdSn5PX-Kg2_$b#d+M_lv~YSyOLPKRxo|w*yE%U#-cf{Y}SRLvuCuKY_iE z>h(NKqx^pXy}!YG-~|(>etlmy`d2Hp`}9xHQ-k+I+1OuQx9_e(0r;@LYWH;;^nSY1 zeyM-Ib-=#Gj9vZb-Oz`sRi?Gi*azvN{ji>}AM87!-`K$Zu(7Z2cnJEnsON?Cwcpz> zLLaI(rzfru=Q~Cpq;s|&nr%KmuzIPJlQ!nBpkM2GKOeOE7ol%7E{)`i>2QuVPtZsr zlX31#CGK%Xv#icS4N)i_9UTpiP%@QI9ZDC9sk~DhcSbU~Y^p$;ljJzboO39X8%|`L zWHFa7IEkYOmK#5uNflE`d~q9EIHT!o+DYW|iDOPGTg)G$(R^Y&XJjJ52_-V=`%_-$aX`+Ecr`t}-TQia-9h1aXgrs73dKaeh&hes zQ&rx{lnKv?rA=t$!5XJ%MvwOI?~8Wz#@!TUxvCPRaN*cEtRaheSB}Y3HdjoA4`q*r zhmWQ+$(z$jGOe*hVT{7bV_A5*vY2;!?oH(j>0Gv|;h-;{$|TsqJ{`^!DQsYdi>V1b z^U|X)mrN8B6i$uV#U4w-S6Z%zyXvloJR@*oJUxO@bBKjMVKaamY`9RMFp6&+HPv97 z{nsk~{lHt#Zo7qZp_~sl*F?12d_fz=(?4*=7JoTEiri@}<^1iw{}rFiBSfC7@vj~K+vs17 zTk1o8ZS;p2bh(?`r(zcheRVbout9WuVO*W#g(e`JMg`4=&?vey6KKAu3h literal 0 HcmV?d00001 diff --git a/zbase/libbase91-i386.so b/zbase/libbase91-i386.so new file mode 100755 index 0000000000000000000000000000000000000000..a999f33f44923981fb966b13f68a8e688d0791d1 GIT binary patch literal 14472 zcmeHOeQaCR6~8ZWscGFLEoFSPECvEJe3-P&3Z)x$K3@7k(xy!eg;E|)?AE4_Y{^+D=^H|UvnrUH}&7&P}zu$d+ z!G$uR?VlmN6aC(~=bm%Vz3<%X>s-I{_N$EzilPXMibb)oq-s>I;007I++u-PA(o3} zb9z$)ox=o}5M{#DgR@N3&ocq`q27&h9?j^GC~Tkg2$8+h#B%5w(fA@Xp}!)qbI_lI zo=8I`sHYDijy4t=ArqFa<*j(&Y3MDifR3>A&CuWP7QzBUKv?>1&_nzfv47t3cIXSo z??PV$+k;8I6ucPNu*E0b{XhBI^v=h8lZ#r zXdtM?rCU=F-SJpQFd{nr9Uakju{#zFieQ9hq9fea?&*$t)`$&_n`-N{Ri2fe)pO-4 z+=+s9T?~+8w&(s#%`>pD=Bc|pd@*PKXZgCaaCi|v6)!?2@pHp9kjrzI*OVvM<1>O7 zrpdARrpRHKA;;pGA&2K#^7(LpcJuLJbC6@Moa9{OHrWxdBVb3sj({BjI|6nD>NfTDhW|NJuYRQY(uP^9)_$huSo44`tUC^1F5;U-=sW)@#{ru@cHXuCo((;9T;7Y_ONOx^pTixR31;JMl$6iXN#26 zCv)+o`nS%}`?>a}Hrn{kcOFw^>jxsGUJdpY`epY~=$6u0X)}9r5 zV#XZPlEjS5c#bD><1nk>C{qRgYJ!%Ebjoqk-}F6D=u0b7bexze%M9bNHBX1@F*OTo zab~GLG)tSN?3m*a3PwWBDhK+q29*7O8dG!9>S}!;M=M`WIZ=9aN$JrO`DVLv$>oO< zYEDV?Fw?)YI%>Or0dteVOFA zGOUb`q>FIL6eot2q&mYJ?wQFfvl`DP`esGXf*06i`cfPt$J8{-V=AXqO#ccO-RIQR zDP_F$s2EUBl^!L=^@O}v$50|?^jt38&SswfmmHJnZ46C6YMfNFiM|Q34-T_Qb>hqe z;CqL6D(Zx;PU^miArLle70aFDl!IE-+|O&tPofz!!oY3`2eJ#laqqm%+AI2DH&He zE$4@KOwHhy#Pn;p$S7xIlo?(%ZN*|Tp{8+L=|!dA_yO(INf{YVX5Y+?{skj_lZz6a z=|yKQkt!VXp+aQ|DogaHnHPWQUHE+R_&?qrbSxOjU6edA@_u>c_}=lI$>WuX-PGV_ zCD}BERjZuT4|8dj9(5EAC>Llsnmj&|D^H%RM6{;l1Wf6^tTL)&4ajEBfue+(LC7=f zR9ET2N*VIBi1+DDlmF!~aO3m$UxZn`um83Sc=g5<*`1YEGu5eC!2gOeeAD>(Fm@* zV;EiFQSfhr|MPE#!9(Ed_}ziW9{zejQXbtdl*gRPGMB?KfJ2cRb^M{A3x2%dqQu!y z;@VugFz)CT>#tmQ`wXkwd&=SY>j`Od);{ei^eOMENRk#+OutXkQC{FM|N0el0l1 zKvKl#WZrq@oV`ojk9uqD3ZfN+%p84_N<}XUWnNQa`zKIj{7H(qRD72OjE7W&+4`X@ znHM+M%Rph?dCFADE9dBq0{I%0|56}R{{i6j;v(nvGO@5AKyFlDSs>H?Cdh?(?W-Uc z=CyC8JsMhh@hz0|<62j{77z7A_IlcS1vA;h7t#fw(*oU5 zE#!{`m=XW+=6;ye-@B!zX;U54SPYuFQES*<)1+z}>$jr{DfODVrJlL=8ydIP)--Bc z8yZ^FRx?W;wakE73gWPk8@%;X`f6p6-zo;{HsPg_s8BXCPNAUtzbU9o6a zFc#k@q(R8v9TJ|vz6hL|JRUPO4+mr2;b`PML4z(9?4Z3_=<0|IkBqG+9_&R~PO>K! z4fx}J;R%Mc-7$Y>Pzwdn(vr*;nm-ovV-##*mEqUl8E!|PQTW2eBRepiwRLw3PkXep zGZ=~I7u}~|+%x&P=bmT1e^XJ=j(Y*g16X#2@8L)#E|y&l%74ecM}ORNiRuD9?TEVo z?z^<(UQCRm%sm=KX~U0x3&1^;cE@m$K-nVNlK7e9{!2UVy#&7ptZ}iP2m#!$S>~Qi zRH0nhFGTSOfLAMI8}9i8_k7DAxf6ICpj|D%?+D^@l&R-dNKU&Xz_H5LF?eHu8_Xiw zk)8l7J2!0rFBN%WQ`&wLuC)mAR zVCVAUd$H={_d(yY0Q+aZZZGBuc5Zf-CwHE|2hM(>mP9*#RxP5=f}FFw;%3+lLBX}p k|BvXp4B#4NpBBBmQix7E$dgrE47t#*a<%zA`BA%n1MkMo;{X5v literal 0 HcmV?d00001 diff --git a/z85base91/libbase91.so b/zbase/libbase91-x86_64.so similarity index 87% rename from z85base91/libbase91.so rename to zbase/libbase91-x86_64.so index f4e3e2698dc75624d5cd0e75d61f4706ed077c12..764107e9451877112344eb8840b1eb61c12a2a09 100755 GIT binary patch delta 362 zcmeAu?a{JPyDzRJFZ{7Td$%K*7YO*1V zIOD|0o-Fc=8Jja%j`4W>0BL+J;nB_7tjEC69s0u~`Ot5o;s7R>~ybp7KogMS+f6aTggK+SHGeFcpeCr@q=bYYB}d{oebv2`+^keKAX zzyJU50BL-A2T1U5b3G1GA2!)m=pWPbKa*{R&oNG)%p>B!7&h5gB!%gt*5u71-i%8o z{}<8TtRuRHYm$P8u%5n^wT-Qvy+d?tTzo=xO>Ldh~d9+K;I85lg0FLkp_X#VlPRLrB>^@m62@fRHbCI<+KF*Zz27y8E}_;+%; z@HxhVlg&gNfI{UWDNNGZlkbXnGoG8QDXPuS!0?%0?w3dN8xD`=BOHe}8!8^>nxx<% ztgEMQWo=_?XYUXl6B`$wP+e0yIa)!Mll$ZU|2;rgZkgPu5G-p4asnTt4}uf;=O`Wj5?Di8e9kZJkzio=<|n$AfGcDEoWss WHF>4EKBtZ-P#+NJY<_6IoEZShbZP7W diff --git a/zbase/libz85b-aarch64.so b/zbase/libz85b-aarch64.so new file mode 100755 index 0000000000000000000000000000000000000000..ff8baffdbedad934d241ef59e40a551a12290a28 GIT binary patch literal 69872 zcmeI0e{fXQ701u+@+%~g08x-ovp_6}y8KWiQ7fASNT>;7SX#7GADi75lDa?W?xG1n z#nx%HnOdS9#j#qff0Q~-$F@$3oz_OC)1k#boHBKEM(rlGQ>WFLP=AzFlGk(JyJz$E zN38wz-}f;u=iYPAIrqHJJ@>u4+_j}+tJmWpxqS39n(7e+^}yP>Q`#Uk(rg~7qb0WO zOmQwFpRIC~B1uPA(Pr^IKH-dyPdM>K?(jPL-CByD>MB2)bjE2?=cVO{?uwkF&1SGv z_enioEE}|)NM)<89e>V5Rr`3Q-7S&VI%*OremAn8ou?lESE5E6bm{q~<5Ht}h}4f` zKU*!w@bTTYeHYz(*~K$|^2CMfe?3-v@wHFCcFFg%&v&UQl)G9I<)`ywL|%IdmHwIg zf48*mmaqMWo6}xCs-Pof7TrhZb7kZg%=Yi)zYEGlQqSuzquUH`%v7y#f6C`OoToE&3OA(Mg?%Uiv332bhQB z$X-%YyyVimOYaqzUiP}O&${$_>nb&0&O&)PxY{dn&2j17@oQXqox1I^*rnHxYU}+j zy*r;(F1;=p+kUM}KUb1rS?~A#VPf!CKbY{*@Jga@PYk|rY{X~0JkGjter?X5Y|GV* zZ5sA_>t9y3a@Tn1@O*DD$L|Mw%FChtG|T% zC!BtkbD_7}azUbJ&sf$Xm80aMB$v)%e%lQ5C+6@nmYazFHbN8{Wxr)_w&lX?H}3O1 zMf@Dfx7dCe(W&L$ZKpdEbe_=WSl^R#I*u&$q|h z+!XZCMmxv1+H!~3-j2UCnESb9J^U))emc~($Lnc&w=$TU7(8J!cIW>ZKhwnEaXW5y z5#>e~Qf{o8a@mE!T-HaYXh!gX?4?T2{Y!&6KhaQjC6BKR=IC6@>Ff#~TcK?E8Q;6} zG_x4u!XAE{=g!u%U41ZDPjrSkgoqCPo}Z~!`97q^>v%4XJGzMd>w>xaSU1X2&8_Ac z9gZK9lY4dt5I-%8SR))mY^#kz)Vc;4(4j*s$JzC^*MQ6H^VdfNA?oqqqw z61$%aKF2w(&T>zTnQx3kP#)!LK);4yyY?TQ`Guhxij zKb5Uk>^KHXW|14^d=wX6izc-u6N4x1S}3>nYE80#@&5SeBai$&cyQ!Vk6LHf`tq}$ zI?8LwZ_QI;=Vv>wZO8R$*MWD4=R4)D9nNdWZEx3))Drv5S2e-)q@K-PGdHU>*X!Tu zTK_fNwpr-hIG>?>9q=K&tYlV2Nh}X)REUeupxOP=e@3UUj@KtJF zW+?XRImfux?7pAZTGd>C_;~Oj)!A!JR9Uv(cx@{_v_f1Xwe|ciQFT|%ZMz1zW|i;q zhLI(zuK0T~Jjx8zUgPP?@%_PSM< zeBq0OcLXnPSi$pn>U`IzGpG6{9sAI-_~k*buvF)*+K(*jAj@8sM_I;L9#{Pd%X)>S zy5u|VInYHj`Y-a-Rn^ORgd+&JTb@o)_-{Je-bCzYF zsq*7rwa0k~7YKj=2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?x zfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?x zfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?x zfWZHmfPc0#!HcsT>CyAuuVs~%M`k*W^K}2pN~dw5?vGD6o^`rEN6RHzp4H<^i^kud zw313St|!*hP_NH1)M?RTuhXCF2zpXWodzw|eeFhXSx!LhM#uDct(N++X_4-a&u|)| zZ`XZ!4)L$h_A=k3<>bFg+grLX9LIIPN{{DsU-H!9{&!+hX8zAYKBE`l-<6HzkZx;j zz1qL3ZTqfE{Ohg^tPiaH1ZnulLnq6m?cW5Yvq;vh zP(Pbw{R{PTNb00eUqP}z3iWeII2P(FDJX^Q8t_pS$-Z)_`DxFib}4k%fREL$Xg@VL1MLlKokzUqDo*o_w^BM)dwFw6D$YyF&f>B=u3Kzkq_eZVL6XE2p%o zJZ8`tYf9s^oc&qKQ$4PU#cHsz2Yq25$@WT z6Q{J^9_CRGmFgY)YO0bCPC=ZhJhhsB%8hdC;>0Q4FTb?sozfT9LnSrHi@#vrYT8$n zPyNTvTTRat*`KiIE!C@O%H{tO>%C>(125ZgO4s*Kwx0Jz(LQ~f^;G8lVA=NPm+ZSr zE&vbjui|~ZjP+icZoibi-&V3czcos(Cf55)v_!jDzr2h-IZd3qZ9UhU{t(xC9p`UO zWB*;&mz(#8)7U>hjsB%+;{4UNpQ@j8+_$etD$^H8QbRvQ!m*ffyBWUK=t&T(6Rc*D z&g}2$2}CGrrp>5Q2&GI1jkOD4=THAKlUqDfDa;n)#Ma6!M2WVEv?4-z`BiOYzuW7Q%pFFA`%LBs~L?* ze^_M_j`bZd-Cl8KoQ$|NQ8SW^nudBJHA~vexJM0RZ#cqZJUfh_5@C_?sS?0T!L|~EM4dSrBJ^Y)T>u-xoqUYHrD2t0~G2 zH75RY?iG1V8_N9RFVCOQ{w>;0&c`C%{_2}s?BCBim6sf6 zx-a1;x(B$S<`;kYJs|R0G1Rhn{-dmS`^)bJk-sSNFP{G?9-qgz)Q9}O7@u$)i|wT^ zQk~x{vWxabdrBiS3+X}#sXeI6+#uWMX4wj3#a_H zM~GY5Hs`RRK-hjr>f~R+L7w6Ct1vQLgaOIU{rjlkiI}g^w)#jqtatLP#!wk1YM$;fHw_aeThyCitE6&%sz0 z!w1#MQwRyz#HOuYvF6%_AAfjuGTgY``?o)~lr_FxIis9o6@iHvV-%tEvyJ_;GCEY~ z0_$S8Tlk74xol_KD=%LvUbyS|3$&fYIZ9HYbV~zrJ~%kdu5;Lnq5B**<2ffuDwG$b z0ZD&kn*C*mtwMheJY#_nr;upA$P-@Jhvo@!JYx%8-`f!h>oLO@HFRC*TUr`)e;^vz z8;Ti$XiGy)dn6oa@wK)GY~Q^-dfNe?z9$s+wTBJ{L?GN2@dxyS>(;c24qtnFq)l`L zI@&sWC}IIa?1=^f!XLOD6*Ho3!Ki2twYGU;5zkt&seVg!jlRmW(zAL>FUQ;!tV$ME z7wcqcSUddL?Jq-0{9}tkv$zXRX4(oup*dX9Gq6eA|JGt+pHmiA+Hw^3Oe9QEVpEJ$ zV)IQRG(t!-LmxOKrwKNvG4t`i-n~GXCq_orCGKyz5M#mbVM((|NFiL}+XTq3P4S zn+eiAmb7f%*#_A-)^FLoiw*KJ_KIZ}(^hGpf^8O;4DLB``!6*L&*Aer=F<0K3G}`r zysp1!JY*i3KnKiifj9Afe7tZwr&#dYK~JBtObqs&TZTB1I)lsM^NY|QQO(XA@$sCp zi;2O3)S`i#S<3J`pBRPacWyD|yT)w5(6~MQE)ts0++t2U-HDI#zj%|0Qq2fFJUE1} zC6K50O-@R5m74KMdrtG?<9X?`DjL(@WfKym${{br8L3!FIZx&rdkPM-uw;;ln z8=o@we&jZ_yc@&uk?e#vYI^fbZB$7nw3Ml(22xoGZQS&ZCz1mrSxPeAuVBQ+)Jy1! zu@c%u{LqBxn*BOP6JyJ8cgP8KrBeH4z$E6K?m}B`x@$63Yd4$TNeqc}^o{Nljj5fs zV@OLf(~x$>O86s7AGVvWQe0nQYFGLSk5Mi2ugz?umn+?Nn4D|Q+CobMV%s^`9{-if?u?+oR%2u(jOfA{hs3g2e z(>nnOE*&p0HM>V9KEbpjNIR0#{mZy^MZs(~YUBAwb`X5N^vP7?zbtH!Yq>U>zyu2t zNK;_qf}|HX?+I&RT9Nj=twq$vv4qy@y^Za6_v+}Gox z^1gkuACUsM$toDUy6N;^*6m&u?4Or-r+{! zVqTeA%s4i7{LpB2_wVD{FqUrETDpaNTWD97YpJQlveySmt#j&kH@QAF%dxq6HgPbQ zTm)PMTm)PMTm)PMTm)PMTm)PMK05-q)v$hVU60%HzwVkPUOifzUA}VF>WVdMzp`$< zueHq|*t0hn+PA;GBOK}cMl@!0bw7Kc=U`QJO|4e9Y4esRo~&=!x~*}0Q*(=V$5UT> z`s*gh`#RTiXJHUIu-C))k zZ>p);pq8TYyi8qL;X#eFx~hCtMfv)b<+A5($+=0&5g(+jS^?F*msiAG>9QpK#J3mY z5W_5>z66qZld^z*8DtMZ0nyLta!w>ge9o@r&%6c&Xt7?LwfZ*YTh$3gmA1ur11j++ zDdGX~A{C5J%ED@$uvP1-;`%d(%{6&*ntd6zvtHf$i8d3n81kcO@vQxqq(WKXu$jMP zn!UzhGkzQFJL=(CwC-sW`nokS)W#oJx7M>aVCbD~x)JOO@AtGG5Ug_#-OJ~PPxr?n zde9g4vtGXT>8(`_TWZjX#n!dWdRve5wEo<&K)o-h=s@J#G)irA^ zdP`Mxy@pndV*JN?d{)s{%0l__Oo_dt!C7Fh-%+7I6~yv-yF!1esT^~=58W4y`Y;RjkUH}EIznw2GlE>Wc&q{JWUaB7@U%rbIs#$7 zO+GX6OvIxEc>mmf&ZflCo9}HBYFeh^oX^`>PF!SM74(1LTw^|-sbuDpeoFF_V4kat z<5^4Yht4w?sujbxb~Bh~B;$BClX)gfVk~JVnCC0wc(#&@pv!UDPG-du&tvL5lgZ`K zo#Vn3ET!Tj7>whYPUe{|^HK7A>;*Hf8qCiCaw&BBAIM0?CBU33KU=Ib3olNko*|j{ zC~O(0G6uX`20_BKa(^3A#`U6+yi_L1#6gJfIpTOWlksngWS1Xg;`n;4iK#;Dqn;rd zcN#XwweVBz^SUGxLm}f{g)ZYxs6vpBqfL^G0>Z;xGiEF->Z`7lWeV7#!-Q|UO4!F9DY7? jeg$CeQ42dr(z%s*&oDuTq&g3_GcJEM-Y+r;QpWuo1PoH2 literal 0 HcmV?d00001 diff --git a/z85base91/libz85b.so b/zbase/libz85b-x86_64.so similarity index 100% rename from z85base91/libz85b.so rename to zbase/libz85b-x86_64.so diff --git a/zbase/libz85p-aarch64.so b/zbase/libz85p-aarch64.so new file mode 100755 index 0000000000000000000000000000000000000000..c634c7a144b95dbee37e037b62b75f1075971501 GIT binary patch literal 69816 zcmeI0e{fXQ701u+M)@fTph5|n1!;+Z%MV3@Kgvr2glIs-VAZORO?E@JZg%7DB7vZ$ z7CY18SfY+U>ga&|hdTYkI!+~jSfk^#SpLz{T5Z*CQag28oe8zJtdeZM=e>K9w?A-f zr~h>3T;}E6d+s^sp8L7?zIT^0D3EvlnQ=&7c8j_q0E zTt>#vag`#;MCWK{;~5!s`$tCI_#*54uHIrJr?~yBc2r|VJ!bMUa?mtIPBvy0ko+7o z{l#*Tu@lL*T(6XE*S=}xmmu6NZWQv9xm-+1cA=8Hs?>D*?VClk@ua9cd;_Lc9k+k3)xM=TMJ$M!}kmWX9yPSxI9x7zLscT+Uco{U7f z*-c&Hcs$uo9jR!PBGCzbJl578N+&~?Q&UTGL!-Sav~rD`Ew8s9|M;)Z{BwJ_^#H1} z{~qgq6f^hJqsCFrW58_eKEvgdIj^E$X7sY|qL=ouJg*$`KpftKQr__B8E^a#h3Y`h)(Ox7b#mDjyx3>9=w?uTG!SJ{#?Yp7vC<8$9h9 zXt%lTEMn6~O*xC`g-`SOIXtGsW{KrHNQbB&ukeElAlDK7ZIGyS2!0DbXv&4*HxlqY zOUR3Ti|rQ>9bb6*-V^oHzkOiA>H3FXf&DOi8%COPZH}*h+;_D1=g#~;nMS$U7;ncj z)>_L)Yn&WEY|8b)-icpjpzJXxpj=G@$EpcKFQvZT|*2BY<)_}if!O<)v7J6nDeq68F>bvuj zv3#ql$K^7D~3z&gLNw-9T1j>UWF0Fkx!Phi`Iu{joW$Eo{*8+HT-=doU54I>A(UqFE+ zn9l&)GJ`Dg;d;$fjZYj^<%EI{}mjUz- z-Yv16LzSA2|c-O`E(LQH= z>M@V(I?Nfl3}ejU#cuw13mj~~zN+sG*E#FLaXDX%nMJ*(icU74#ymJT)+5&;)MXy+ zpP>&d=JGmLS~=v%>+9@+(cagceLdQHwCrBvJ<{i+)?waffAK$m=!qxH9I(ecdy8{l zS3xItnEiA=v%-4uTG4#$RoQdMHD+w~u?^zlz51ra4uX%ff6ad5{Wm;@ylaqeP2rx6 zvp?p=Ht(=aHEgOqdldG6fS$01Vc6!ul-Dr7+g&eQ5O}@D&j`LJ2bgD&_ZT=1A%CZi zITW8|>Iokm=JRdVdI)~dowD{(_0Vo>0DjZ57>DB(uT_lK3gdF#L&%rMU>~dr_L8wj zu1;>4AMZt8!#vJ=9%r1-QXc1<^XK_HX9{}-agGn+46}WMA7_f+1)eeZOgU!;``a>S zX5pg2dCs}#`SSY&%s8KVya$kv`wrkeh4FYlIcJ9R=h&=AK0~N;9z1R%pC`=eZe9=G zYsd$2oV6S6JvROhmG6;!iwNbX`{z5K*N<;^FDh#>K`wmX;et3$S+R1}rI)R~d`;cj za9evM+Oe}U_N_bOU5RA(ovCzY*Im1N_Ux^1Xk5QxW7DSQYp!kCyyd#B*KfO_b^DFq zxasCwLYG{z^y=FeEMKzjYgc`}_Tq2e)_b@0)kPO^{OZ7!>mm8u*`3ShZ-qPr*$w$F zpSs*m7#4tnSAS3l#R@2Oc&&-6d>{(GkX1@xc(e7wKU_f*x>m8KtET9kkiPy$Lo z2`B+2pahhF5>Nt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KnW-TC7=Y9fD%vwNNt4KneWc5a>JGO?8=(K9l(aMou@f_AIw@j%gpA=2pI9+9RW`XSHch zHgcYkr%eC+qW(|E@<~>Wn1RhOWCdJ9lNJU2ZhNXL=&+F{4Vr7(#*IGAy8(?G?J)i4 z8ac_xIYu6ML~ck?G>pEqqY9U3w1VUt+Sv?Wgy?tdr7WaR%Y zq-7T1OfpC=(Wb`6D}uF6Tee>uTzOe&RcOVR7%o6QI#wpV0AkAed&2}MFuv}E`ik-W zQK&zQgma-DyU46hp?(s{Iv46ElkA5={S=aOP^h0u!mCg}jSk6%_H+bjI?4I*s4?4f z=ndJ>o{j*`psjKc3-p!rcv*cF$$9jcBmPX1b6BXKMO0?r1ZXx5nsZcWfA0ACD%77x zvOfy-=ToiOH-&mReG^*V9u;&lKcR7J?l~;w$+-M9p_lXR+Lx}Et1sox=lRUzPNBM@ zb(A=-6|Gmz7si=JgGF(cnEZ?N%Zz@c$bKF4M5UY?jJ>?a9W(s736GB^_{;HgKL9;*mhT0cPI2Ru?zhL#>!VV=YhOt}DcZNLej-m3o_>Z(xpjHTiGMz=kbPy& zx5OFOn?SC<^nAVM#F^078hdwrr_p}-02Rckq$>Fk8hs69xjfx*E2*={{&m>tX1F zrD5nf$G)^<@C^3Up31~RNm|rH?csRb-W?6!VRs~0oq!r5o!QmV5o)JMG!@+$OJ|}f zJJV&i$CHU@nifUKwj)Vn1H&VK#ZuRE+tz_4&PQzp4w))K*>`Pn~cMz_1mCo-P|ayO)b|o)VJ8zZQOXn zhE}_^zM*9U0&)&yH@9xKCqTf7F|>x;5I>q|Pe!6Pr-pd#Jz>rv9FOgdT1_p@4UP7y z(8@K`1<1*EuO|lYplyGby*{u4pP9OC|~$Hu_T0 zc$ghbtvjBfkV7hziT0q*i;TWxB%BFTDB5Y3wKD==X}KQm`nn!cN8oT*tR15!5et7p z&H!$(wse|8*k)bWJ*E5oKh^NPpYP7*_>SN6;xG4j%jji%%M2v)F#5g4gd10FLXGeK;xG45 zkur|tFLolg!=HakNZU!2qZ?2`uA^|1$CmtA$0dS?`@}yeenuW)BNQ25{3Wjh`q;FA zjZozND{_aUb^ZB{RqWpb9p@#VOSvbDlyOWLXDq3+OylwT%Y9np4mR?XDo>rO_u%pR zA2RnQk#i+6;}c7Cp~$>tT-gW5(Z?p@FZY$fJbMxDHHAg|MZSSw4S1DejFCVrWg_n$h&wfxu{ J6!l{Ne*?~~Mx6iv literal 0 HcmV?d00001 diff --git a/zbase/libz85p-i386.so b/zbase/libz85p-i386.so new file mode 100755 index 0000000000000000000000000000000000000000..12eebca7f067d1a7383bcf62688144599e6bb5be GIT binary patch literal 14484 zcmeHOeQ;FO6~8YD5MX6fj8=upN-5>bHGx10peBSDKoSTf5e36`liiTjBun=r0=gD> zV0nF{V%1JNc1EVQ|8>S%8OAD(2|*GWoLNZ3Rikb(b{0tM0@bz)`7s8-yelWWYZ#wf>M@h@pE7%t42iq4MZteVB@@yMXr^7==uv zo<4{H)G@IPGSSpkeim^38hR6FKu0w770~an31Q+g=!m9%E%Xq-0c@W`UI%?@|Fh7R zj?jVC{p5loY-0K9YH@qPk{6aZ%7TmL%}x9{KK-_7*1XAV>lPqmMjJ^;Z7drL%t-R| zMIU2;hD<=+uvDds<+v;@Gdzzh%p6zboY4!v>I_ucJ zaK59-xkb2}>%1;E#Vw-A+1Ti<6ZJl~Te#dK<&B=&I=kO%Um%v3tz25-m}}3s&l~3R z&~piS$Yd^3{xZZqXw10!r+ARAO!2E^S&qkZ1GD!#$OO)B=15>3lV%qgv=8!V#3V+F z1ox97xp7iXGT4nl46{f%DbkRRKso~H2&5yBjzBsB=?J7Fkd8n)0_g~RiwIOI{n@eQ zFx=U39$RWnxtvHO_Ot~u5@F?N_QsCHE8JitUexy~I){ClzLOJYqcBap$lXo@)n~&H zwaL>u>i3WQ#rmMxdE_bhZv8now5_{%E$j+>PumqY+Pbk?{Y>f4UCSnO-tMrs2BwRS z)`QbwCt|}sWCQiYj zRx?DOhP(ZjthawjN3lv6?(7)E&kXRhXm{ z6kaS_{A*LoP(wox4!2s8F50AaP0p6OtcQg4Q2bT?s4-h^YsZ+|yZ@%f)nlo~QK@U} zMKwo}G2BpFX3Nuxd{~0PVmV^nRfkk`%16;%bf=5mIlB`VP3}tVuO?KvuTwrj{rL@A zuDq@eLZ;b6R^}SYwNmVaOVe6r6 zO+MOMA*t0zwd!8v=lEXq?*#gHBCF+qB6p#GU3&lgoJ$OFH1;y)DqLxDj#hnC1?_Eh z4{Y4zsNpQY94Lsr1dE*=m6&l#$RYw$v~tU9@0>VsLhTwn(|#@^nf96O>T%_&6iDXz z89mS0dMMjeIi`eSSuIK3qq@$b#3bF%;2F5!B->O(T-%s5o(l*4+Ay`6FonNB&P%f(uC*yui{V?+ugcANl+Bql(c zwp-z?p<+@*uFZ(`rFKc~QBXw*MY3A{f(bpG*8vH>dejUBcL>TC%l$sJQ|>jIjq1%l z*eP>866Z^e0ne6Wg>tXI0FyYy4JK!D3icUO@SJ***qjUX*_duG{lZsqw)Jywm#BK`_H@CGD@gW<&LvSG%LC992)Mk?t7t z6CK-*`?EZ%Ug`c+J&CfKUiFGr(}&ykWcT^8mvEz8R3n4IsCruM7Mnxh4?wGdXsT*eDcTt&og zFZ{&60YkZ@hV!xf&-r@1I{g+kZkE^Yx4zM(T$rMkM}l(UW$UhYF~-80s8!+8fPoo482p3{^zO}VQXu-&vdY}f?3u>shJ8C^t-m8An{W+o%Vz9z$d5?Ob`%}HV zCX9$(=su$h9qMIDW5epz?lXv!`kwm1V6aC$Ip_`}jNOGNeWQaD3OoJdX3OD8)jeuA zvK5M`c%+o~A`>&?w|<~@BMn!&W8LS*Mn<}h=ahOyy*TrOTiffc%}0Q5*IR4BpO&Lq zFsg<6&gLM}`u_rp@ejbxj!%BbWe(4Ogdq*-2&5yBjzBsB=?J7F@P8J8mv>L*ZcF@q z6W=W3pP%GgV!pd2@P9@^UjE#91@jmDXknqVw$A0Q-`wDN_>sn@X784t`22xj%cEPj zZ7*J0vP>>rzGCHltIEn(uc=tOuCl87{`C($xWRtsqS<$Ey5p`{_fB7O&y1NHYaR;~ z{b1Vn;a8fJxwr_F@30Sio=7|d`VM{<*a2$2kVqT=z3EaS!EMApun`{s6<;J0+2~l_ zKNE>6Q0~$R()M*idOTa2WXa0fi*g&vIGiQ$n+GZ;WS35`+-Dsh$l5CIy>0QVxznan zUj&rV=2B2mJRy5e#?l)`J@FR281J-yjN;AE^S$-3zVv6APi85jGOtJ-#m9;}iw$$V z&HXUD`(h%&_u9ku%aqK_lhRuq@S20-gTRXz7n;yv1+Lt##!)tv_2gtflI^(1jWn}r z{OS~G`S^^|Yeew?s^`K-590OEfAn=7lQ#M1Hbn}&1*(h*2UARU2p z1kw>mM<5-6bOh27NJrp*F#<(6iyX&hj9a9AKnn)}8HaR*pA>txVhLIyJ2Zm{~*hU-Vb)U)kp zb&YXhUM~Yv&${z_028KynUW&YJ`Zy0+4+(by$$+xUy}t-v)k*$hK2LXiz|d<;e0=i z({Ek4z`ofXaBQh_1R8?PkJxLsqBPj#b-6tA1P{r3u4m=atz4cK-OAzedmRnVW*3j$ zf77KC#>*aDU0lAh1ZvD4hrGg3x~{lfcC1*o4n;V(?~qq76ME~C&_D?;q&y)H^=9&DwYtHODt~vQXY{Y~6#xeilqP**U7I3|<53c)# zfw23xH3gitpaGvj8%(a*8*tk9G&ubY!tUDE z3}*%n_zX>p+voRqo39ZZ(D~erv^SV7jR9fTBWDk|w}RJ`Y4>?u&VWvB zXmFvXDH$~!PM^<-Ua*GA!>_Z+Q-?Oa@P&(AZ@_R?>-P(LowupU-5kK&`?kP!lfUb6 z-7~+%nU+$S_dWq+xR`du;PI26m;Tx?|A})_^t2 z!Rj`=l`DZzCOtmJ=I*T(ik@-6rONPi+^Xcob-S z$78JfT>H81oBl|-p6&qBZYhw@2_pMVJ(o7qP6f}t^7*41z{o~E3A7{Z2AXy@+5q!( zjd6jf`+O?Jj%zgWFcX;s+7X@un*O*}6B!p1%yyqg0qqzUy<;{Zo*@!l(+JeG`xgpCHg*62 literal 0 HcmV?d00001 diff --git a/z85base91/libz85p.so b/zbase/libz85p-x86_64.so similarity index 100% rename from z85base91/libz85p.so rename to zbase/libz85p-x86_64.so diff --git a/zbase/version.py b/zbase/version.py new file mode 100644 index 0000000..76c4342 --- /dev/null +++ b/zbase/version.py @@ -0,0 +1,6 @@ +# START_VERSION_BLOCK +VERSION_MAJOR = 0 +VERSION_MINOR = 0 +VERSION_BUILD = 1 +VERSION_ALPHA = 0 +# END_VERSION_BLOCK diff --git a/zbase/z85b.py b/zbase/z85b.py new file mode 100644 index 0000000..50c89a0 --- /dev/null +++ b/zbase/z85b.py @@ -0,0 +1,108 @@ +""" +Python implementation of Z85b 85-bit encoding. + +Z85b is a variation of ZMQ RFC 32 Z85 85-bit encoding with the following differences: +1. Little-endian encoding (to facilitate alignment with lower byte indices). +2. No requirement for a multiple of 4/5 length. +3. `decode_z85b()` eliminates whitespace from the input. +4. `decode_z85b()` raises a clear exception if invalid characters are encountered. + +This file is a derivative work of https://gist.github.com/minrk/6357188?permalink_comment_id=2366506#gistcomment-2366506 + +Copyright (c) 2013 Brian Granger, Min Ragan-Kelley +Distributed under the terms of the New BSD License. +""" +import re +import struct +from typing import Union + +from hivemind_bus_client.exceptions import Z85DecodeError + + +class Z85B: + # Z85CHARS is the base 85 symbol table + Z85CHARS = bytearray(b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#") + + # Z85MAP maps integers in [0, 84] to the appropriate character in Z85CHARS + Z85MAP = {char: idx for idx, char in enumerate(Z85CHARS)} + + # Powers of 85 for encoding/decoding + _85s = [85 ** i for i in range(5)] + + # Padding lengths for encoding and decoding + _E_PADDING = [0, 3, 2, 1] + _D_PADDING = [0, 4, 3, 2, 1] + + @classmethod + def encode(cls, data: Union[str, bytes], encoding: str = "utf-8") -> bytes: + """ + Encode raw bytes into Z85b format. + + Args: + data (Union[str, bytes]): Input data to encode. + encoding (str): The encoding to use if `data` is provided as a string. Default is 'utf-8'. + + Returns: + bytes: Z85b-encoded bytes. + """ + if isinstance(data, str): + data = data.encode(encoding) + data = bytearray(data) + padding = cls._E_PADDING[len(data) % 4] + data += b'\x00' * padding + nvalues = len(data) // 4 + + # Pack the raw bytes into little-endian 32-bit integers + values = struct.unpack(f'<{nvalues}I', data) + encoded = bytearray() + + for value in values: + for offset in cls._85s: + encoded.append(cls.Z85CHARS[(value // offset) % 85]) + + # Remove padding characters from the encoded output + if padding: + encoded = encoded[:-padding] + return bytes(encoded) + + @classmethod + def decode(cls, encoded_data: Union[str, bytes], encoding: str = "utf-8") -> bytes: + """ + Decode Z85b-encoded bytes into raw bytes. + + Args: + encoded_data (Union[str, bytes]): Z85b-encoded data. + encoding (str): The encoding to use if `encoded_data` is provided as a string. Default is 'utf-8'. + + Returns: + bytes: Decoded raw bytes. + + Raises: + Z85DecodeError: If invalid characters are encountered during decoding. + """ + # Normalize input by removing whitespace + encoded_data = bytearray(re.sub(rb'\s+', b'', + encoded_data if isinstance(encoded_data, bytes) + else encoded_data.encode(encoding))) + padding = cls._D_PADDING[len(encoded_data) % 5] + nvalues = (len(encoded_data) + padding) // 5 + + values = [] + for i in range(0, len(encoded_data), 5): + value = 0 + for j, offset in enumerate(cls._85s): + try: + value += cls.Z85MAP[encoded_data[i + j]] * offset + except IndexError: + break # End of input reached + except KeyError as e: + raise Z85DecodeError(f"Invalid byte code: {e.args[0]!r}") + values.append(value) + + # Unpack the values back into raw bytes + decoded = struct.pack(f'<{nvalues}I', *values) + + # Remove padding from the decoded output + if padding: + decoded = decoded[:-padding] + return decoded diff --git a/zbase/z85p.py b/zbase/z85p.py new file mode 100644 index 0000000..3298a17 --- /dev/null +++ b/zbase/z85p.py @@ -0,0 +1,88 @@ +from typing import Union +import struct + +class Z85P: + """ + Z85 is a class that provides encoding and decoding methods for transforming raw bytes into the Z85 encoding format. + Z85 encoding represents 32-bit chunks of input bytes into a base85-encoded string with padding applied. + The padding is added to ensure the encoded data's length is a multiple of 4 characters. + The first byte of the encoded data indicates how many padding characters were added, which can be removed during decoding. + """ + Z85CHARS = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#" + Z85MAP = {c: idx for idx, c in enumerate(Z85CHARS)} + + _85s = [85 ** i for i in range(5)][::-1] + + @classmethod + def encode(cls, rawbytes: Union[str, bytes]) -> bytes: + """ + Encodes raw bytes into Z85 encoding format with padding, and prepends the padding size. + + Args: + rawbytes (Union[str, bytes]): The input raw bytes to be encoded. + + Returns: + bytes: The Z85-encoded byte sequence with appropriate padding and padding size indication. + + Notes: + The padding is applied to ensure the length of the encoded data is a multiple of 5. The first byte in the + returned byte sequence represents the number of padding characters added. + """ + if isinstance(rawbytes, str): + rawbytes = rawbytes.encode("utf-8") + + padding = (4 - len(rawbytes) % 4) % 4 # Padding to make the length a multiple of 4 + rawbytes += b'\x00' * padding + + # The first byte indicates how many padding characters were added + nvalues = len(rawbytes) // 4 + values = struct.unpack('>%dI' % nvalues, rawbytes) + encoded = [padding] + + for v in values: + for offset in cls._85s: + encoded.append(cls.Z85CHARS[(v // offset) % 85]) + + return bytes(encoded) + + @classmethod + def decode(cls, z85bytes: Union[str, bytes]) -> bytes: + """ + Decodes a Z85-encoded byte sequence back into raw bytes, removing padding as indicated by the first byte. + + Args: + z85bytes (Union[str, bytes]): The Z85-encoded byte sequence to be decoded. + + Returns: + bytes: The decoded raw byte sequence with padding removed. + + Raises: + ValueError: If the length of the input data is not divisible by 5 or contains invalid Z85 encoding. + + Notes: + The first byte of the encoded data indicates the padding size, and this padding is removed during decoding. + """ + if isinstance(z85bytes, str): + z85bytes = z85bytes.encode("utf-8") + + if len(z85bytes) == 0: + return z85bytes + + if len(z85bytes) % 5 != 1: + raise ValueError('Invalid data length, should be divisible by 5 with 1 extra byte for padding indicator.') + + padding = z85bytes[0] # Read the padding size from the first byte + if padding < 0 or padding > 4: + raise ValueError('Padding size must be between 0 and 4.') + + z85bytes = z85bytes[1:] # Remove the first byte (padding size byte) + + values = [] + for i in range(0, len(z85bytes), 5): + value = 0 + for j, offset in enumerate(cls._85s): + value += cls.Z85MAP[z85bytes[i + j]] * offset + values.append(value) + + decoded = struct.pack('>%dI' % len(values), *values) + return decoded[:-padding] if padding else decoded # Remove padding