Merge pull request #1 from rapiddweller/extend-cli

Extend CLI and Add Demos to DATAMIMIC CE

.github/workflows/main.yml

@@ -81,6 +81,51 @@ jobs:
           uv sync --all-extras
         shell: bash
 
+  build:
+    name: Build for ${{ matrix.os }} ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        arch: [ x64, arm64 ]
+        exclude:
+          - os: windows-latest
+            arch: arm64  # Windows GitHub runners do not support ARM64 yet
+          - os: ubuntu-latest
+            arch: arm64  # Ubuntu GitHub runners do not support ARM64 yet
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+
+      - name: Set Up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          architecture: ${{ matrix.arch }}
+
+      - name: Install uv Package Manager
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: |
+          uv sync
+        shell: bash
+
+      - name: Build Package
+        run: |
+          uv build
+        shell: bash
+
+      - name: Upload Build Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-files-${{ matrix.os }}-${{ matrix.arch }}
+          path: dist/*
+
   unit_test:
     runs-on: ubuntu-latest
     needs: install_dependencies
@@ -152,6 +197,7 @@ jobs:
 
   integration_test:
     runs-on: ubuntu-latest
+    needs: install_dependencies
     services:
       postgres:
         image: postgres:13
@@ -171,7 +217,6 @@ jobs:
           - 27017:27017
         options: --health-cmd "mongo --eval 'db.stats().ok'" --health-interval 10s --health-timeout 5s --health-retries 5
     container: python:3.11-slim-buster
-    needs: install_dependencies
     steps:
       - name: Setup git
         run: |
@@ -196,61 +241,49 @@ jobs:
   publish_dev:
     if: github.ref == 'refs/heads/development'
     runs-on: ubuntu-latest
-    needs: [ unit_test, functional_test, integration_test ]
+    needs: [ build, unit_test, functional_test, integration_test ]
     environment:
       name: testpypi
       url: https://test.pypi.org/p/datamimic_ce
     permissions:
       id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
       contents: read
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
         with:
-          fetch-depth: 0
-          fetch-tags: true
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-      - name: Package
-        # IMPORTANT: Use "pip install build" instead of "uv add" to avoid updating pyproject.toml, which accidentally create dirty package version
-        run: |
-          uv venv .venv --python 3.11
-          . .venv/bin/activate
-          which python
-          uv pip install build
-          python -m build
-        shell: bash
+          name: 'dist-files-*'
+          path: dist
+
       - name: Publish distribution to TestPyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
-  #          verbose: true
+          packages: 'dist/*'
 
   release:
     if: startsWith(github.ref, 'refs/tags/')
-    needs: [ unit_test, functional_test, integration_test ]
     runs-on: ubuntu-latest
+    needs: [ build, unit_test, functional_test, integration_test ]
     environment:
       name: pypi
       url: https://pypi.org/p/datamimic_ce
     permissions:
       id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
       contents: read
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-      - name: Package
-        run: |
-          uv venv .venv --python 3.11
-          . .venv/bin/activate
-          which python
-          uv pip install build
-          python -m build
-        shell: bash
-  #      - name: Publish distribution to TestPyPI
-  #        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: 'dist-files-*'
+          path: dist
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages: 'dist/*'
 
   trigger_ee_dev:
     if: github.ref == 'refs/heads/development'
@@ -262,4 +295,78 @@ jobs:
         run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline;
         env:
           EE_TOKEN: ${{ secrets.EE_TOKEN }}
-        shell: bash
+        shell: bash
+
+  e2e_test:
+    name: E2E Test on ${{ matrix.os }} ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    needs: build
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        arch: [ x64, arm64 ]
+        exclude:
+          - os: windows-latest
+            arch: arm64  # Exclude Windows ARM64 as it's not supported
+          - os: ubuntu-latest
+            arch: arm64  # Ubuntu GitHub runners do not support ARM64 yet
+    steps:
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          architecture: ${{ matrix.arch }}
+
+      - name: Install uv Package Manager
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist-files-${{ matrix.os }}-${{ matrix.arch }}
+          path: dist
+
+      - name: List Contents of dist Directory
+        run: |
+          if [ "$RUNNER_OS" == "Windows" ]; then
+            dir dist
+          else
+            ls dist
+          fi
+        shell: bash
+
+      - name: Install Built Package on Windows
+        if: runner.os == 'Windows'
+        run: |
+          $wheel = Get-ChildItem -Path dist\*.whl | Select-Object -First 1
+          pip install $wheel.FullName
+        shell: pwsh
+
+      - name: Install Built Package on Unix
+        if: runner.os != 'Windows'
+        run: |
+          pip install dist/*.whl
+        shell: bash
+
+      - name: Run DATAMIMIC Help Command
+        run: datamimic --help
+
+      - name: List Available Demos
+        run: datamimic demo list
+
+      - name: Create All Demos
+        run: datamimic demo create --all --target my_demos
+
+      - name: Run the demo-model Demo
+        run: datamimic run ./my_demos/demo-model/datamimic.xml
+
+      - name: Run the demo-condition Demo
+        run: datamimic run ./my_demos/demo-condition/datamimic.xml
+
+      - name: Run the demo-custom-components Demo
+        run: datamimic run ./my_demos/demo-custom-components/datamimic.xml
+
+      - name: Run the demo-datetime Demo
+        run: datamimic run ./my_demos/demo-datetime/datamimic.xml

.gitignore

@@ -54,4 +54,9 @@ tests_ce/functional_tests/test_external_source/conf/
 **/exported_data/
 
 # UV
-uv.lock
+uv.lock
+
+#datamimic
+datamimic.log
+datamimic.log.*
+exported_data/

README.md

@@ -11,12 +11,12 @@
 - [What is DATAMIMIC?](#what-is-datamimic)
 - [Key Features](#key-features)
 - [Getting Started](#getting-started)
-  - [Prerequisites](#prerequisites)
-  - [Installation using uv Package Manager](#installation-using-uv-package-manager)
+    - [Prerequisites](#prerequisites)
+    - [Installation using uv Package Manager](#installation-using-uv-package-manager)
 - [How to Contribute](#how-to-contribute)
 - [License](#license)
-  - [Attribution](#attribution)
-  - [Author's Note](#authors-note)
+    - [Attribution](#attribution)
+    - [Author's Note](#authors-note)
 - [Acknowledgments](#acknowledgments)
 - [Support](#support)
 - [Stay Tuned!](#stay-tuned)
@@ -27,17 +27,24 @@
 
 ## Introduction
 
-Welcome to the official GitHub repository for **DATAMIMIC Community Edition**! Our platform revolutionizes test data generation by leveraging AI and model-driven technologies. **DATAMIMIC** ensures robust data privacy through sophisticated anonymization and pseudonymization techniques, making it an indispensable tool for developers and testers aiming for compliance and efficiency.
+Welcome to the official GitHub repository for **DATAMIMIC Community Edition**! Our platform revolutionizes test data
+generation by leveraging AI and model-driven technologies. **DATAMIMIC** ensures robust data privacy through
+sophisticated anonymization and pseudonymization techniques, making it an indispensable tool for developers and testers
+aiming for compliance and efficiency.
 
 [![Watch the video](https://datamimic.io/wp-content/uploads/sites/7/2024/04/datamimic_cover_play.jpg)](https://www.youtube.com/watch?v=sycO7qd1Bhk)
 
 ---
 
 ## What is DATAMIMIC?
 
-**DATAMIMIC** is an advanced Python-based platform designed to create scalable and reliable test data using AI and model-driven methodologies. By simulating real-world data scenarios, it enables comprehensive testing without compromising sensitive information. **DATAMIMIC** ensures efficiency and compliance with global data protection standards like GDPR, making it ideal for industries such as finance, healthcare, and e-commerce.
+**DATAMIMIC** is an advanced Python-based platform designed to create scalable and reliable test data using AI and
+model-driven methodologies. By simulating real-world data scenarios, it enables comprehensive testing without
+compromising sensitive information. **DATAMIMIC** ensures efficiency and compliance with global data protection
+standards like GDPR, making it ideal for industries such as finance, healthcare, and e-commerce.
 
-**Note:** The Community Edition does not include the Rust core, which is part of the Enterprise Edition, but still offers a robust set of features for non-commercial use.
+**Note:** The Community Edition does not include the Rust core, which is part of the Enterprise Edition, but still
+offers a robust set of features for non-commercial use.
 
 ---
 
@@ -61,7 +68,8 @@ Welcome to the official GitHub repository for **DATAMIMIC Community Edition**! O
 
 ### Installation using uv Package Manager
 
-We recommend using the [`uv`](https://github.com/astral-sh/uv) package manager for setting up **DATAMIMIC**. `uv` simplifies virtual environment and dependency management.
+We recommend using the [`uv`](https://github.com/astral-sh/uv) package manager for setting up **DATAMIMIC**. `uv`
+simplifies virtual environment and dependency management.
 
 #### Steps:
 
@@ -85,37 +93,77 @@ We recommend using the [`uv`](https://github.com/astral-sh/uv) package manager f
    cd datamimic
    ```
 
-4. **Initialize uv Environment**:
+4. **Synchronize Dependencies**:
 
    ```bash
-   uv init
+   uv sync
    ```
 
-   This will create a new virtual environment and a `uv.lock` file if it doesn't already exist.
+   This command installs all the dependencies specified in the `pyproject.toml` file.
 
-5. **Synchronize Dependencies**:
+5. **Run DATAMIMIC**:
 
    ```bash
-   uv sync
+   uv run datamimic --help
    ```
 
-   This command installs all the dependencies specified in the `pyproject.toml` file.
+   This command will display the available options and commands for **DATAMIMIC**.
 
-6. **Activate the uv Environment**:
+6. **List available demos**:
 
    ```bash
-   uv
+   uv run datamimic demo list
    ```
 
-   This command activates the virtual environment. You should see `(uv)` at the beginning of your terminal prompt.
+   This command will display the available demos for **DATAMIMIC**.
 
-7. **Run DATAMIMIC**:
+7. **How to access a demo**:
 
    ```bash
-   run_mimic
+   uv run datamimic demo create <demo_name>
    ```
 
-   This will execute the `run_datamimic` script defined in your `pyproject.toml`.
+   Replace `<demo_name>` with the desired demo name to run the demo or leave it blank to run the default demo (
+   demo-model).
+
+    ```bash
+    uv run datamimic demo create --all --target my_demos
+    ```
+
+    This command will create all demos in the `my_demos` directory.
+
+8. **Run the default demo**:
+
+   ```bash
+    uv run datamimic run ./demo-model/datamimic.xml
+    ```
+   This command will run the default demo (demo-model) for **DATAMIMIC**.
+
+    ```bash
+    Environment properties file not found. Continuing with empty properties.
+    2024-11-07 15:56:46,046 | INFO | DATAMIMIC | Starting DataMimic Process
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | DATAMIMIC lib version: 0.9.1.dev2
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | System name: Darwin
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | Python version: 3.11.9 (main, May 14 2024, 04:42:59) [Clang 15.0.0 (clang-1500.1.0.2.5)]
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | Number of CPU cores: 8
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | CPU architecture name: arm64
+    2024-11-07 15:56:46,047 | INFO | DATAMIMIC | RAM size: 16.00 GB
+    2024-11-07 15:56:46,064 | INFO | DATAMIMIC | Using default page size 16 for processing by page
+    2024-11-07 15:56:46,064 | INFO | DATAMIMIC | Processing 1 pages for 16 products of 'CARS'
+    2024-11-07 15:56:46,065 | INFO | DATAMIMIC | Processing 16 product 'CARS' on page 1/1 in a single process
+    2024-11-07 15:56:46,065 | INFO | DATAMIMIC | Process product 'CARS' with single process
+    2024-11-07 15:56:46,065 | INFO | DATAMIMIC | Generating 15 records 'CARS' takes 0.00022 seconds (66717 records/second)
+    2024-11-07 15:56:46,065 | INFO | DATAMIMIC | Consuming 15 records 'CARS' takes 3e-05 seconds (537731 records/second)
+    2024-11-07 15:56:46,083 | DEBUG | DATAMIMIC | Processing temp result directory has been removed successfully.
+    2024-11-07 15:56:46,089 | INFO | DATAMIMIC | Using default page size 10 for processing by page
+    2024-11-07 15:56:46,089 | INFO | DATAMIMIC | Processing 1 pages for 10 products of 'customer'
+    2024-11-07 15:56:46,089 | INFO | DATAMIMIC | Processing 10 product 'customer' on page 1/1 in a single process
+    2024-11-07 15:56:46,089 | INFO | DATAMIMIC | Process product 'customer' with single process
+    2024-11-07 15:56:46,190 | INFO | DATAMIMIC | Generating 10 records 'customer' takes 0.10123 seconds (98 records/second)
+    2024-11-07 15:56:46,190 | INFO | DATAMIMIC | Consuming 10 records 'customer' takes 3e-05 seconds (322638 records/second)
+    2024-11-07 15:56:46,209 | DEBUG | DATAMIMIC | Processing temp result directory has been removed successfully.
+    2024-11-07 15:56:46,209 | INFO | DATAMIMIC | Reverted working directory to: /projects/datamimic
+    ```
 
 ---
 
@@ -236,4 +284,5 @@ The `pyproject.toml` file defines the build system requirements and project meta
 - **Version**: `0.1.1a1`
 - **Python Version Required**: `>=3.10`
 - **Entry Point**: `run_mimic` script to execute `datamimic.data_mimic:run_datamimic`
-- **Dependencies**: A comprehensive list of packages required for **DATAMIMIC** to function properly. Ensure all dependencies are installed during setup.
+- **Dependencies**: A comprehensive list of packages required for **DATAMIMIC** to function properly. Ensure all
+  dependencies are installed during setup.