From 5f4b4aed4217068dbb9c10386d1f3ac17892d3dd Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 11:38:31 -0500 Subject: [PATCH 1/7] feat: remove gramine config files and simplify Dockerfile --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8d30e06..d6d560e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,5 @@ FROM python:3.12-slim -# Install any Python dependencies your application needs, e.g.: -RUN pip install --no-cache-dir requests - -RUN mkdir /sealed && chmod 777 /sealed - WORKDIR /app COPY . /app From 70b147ce5a47d64c41db37c83d3c085ce3178c78 Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 12:22:27 -0500 Subject: [PATCH 2/7] feat: Remove GSC configuration file which is unused without gramine --- config.yaml | 55 ----------------------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 config.yaml diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 04270db..0000000 --- a/config.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# NOTE: This file was copied from https://github.com/gramineproject/gsc/blob/fcf96546f4a23a4e6bcc6a14d80cf1521c018fc9/config.yaml.template - -# -# Specify the OS distro that is used to build Gramine, i.e., the distro from where the Gramine build -# gets all tools and dependencies from. This distro should match the distro underlying the -# application's Docker image; otherwise the results may be unpredictable (if you specify `"auto"`, -# which is recommended, you don't need to worry about the mismatch). -# -# Currently supported distros are: -# - ubuntu:20.04, ubuntu:21.04, ubuntu:22.04, ubuntu:23.04 -# - debian:10, debian:11, debian:12 -# - centos:8 -# - quay.io/centos/centos:stream9 -# - redhat/ubi8:8.8, redhat/ubi9:9.4 -# - redhat/ubi8-minimal:8.8, redhat/ubi9-minimal:9.4 - -# If Distro is set to "auto", GSC detects the distro automatically by examining the supplied -# Docker image. Alternatively, Distro can be set to one of the supported distros mentioned above. -Distro: "auto" - -# If the image has a specific registry, define it here. -# Empty by default; example value: "registry.access.redhat.com/ubi8". -Registry: "" - -# If you're using your own fork and branch of Gramine, specify the GitHub link and the branch name -# below; typically, you want to keep the default values though. -# -# It is also possible to specify the prebuilt Gramine Docker image (that was built previously via -# the `gsc build-gramine` command). For this, remove Repository and Branch and instead write: -# Image: "" -# -# GSC releases are guaranteed to work with corresponding Gramine releases (and GSC `master` -# branch is guaranteed to work with current Gramine `master` branch). -Gramine: - Repository: "https://github.com/gramineproject/gramine.git" - Branch: "master" - -# Specify the Intel SGX driver installed on your machine (more specifically, on the machine where -# the graminized Docker container will run); there are several variants of the SGX driver: -# -# - upstream (in-kernel) driver: use empty values like below -# Repository: "" -# Branch: "" -# -# - DCAP out-of-tree driver: same as above, use empty values -# Repository: "" -# Branch: "" -# -# - legacy out-of-tree driver: use something like the below values, but adjust the branch name -# Repository: "https://github.com/01org/linux-sgx-driver.git" -# Branch: "sgx_driver_1.9" -# -SGXDriver: - Repository: "" - Branch: "" \ No newline at end of file From 52c5fd30a55cb401f24c6958081f7319fbcaac0a Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 12:22:58 -0500 Subject: [PATCH 3/7] feat: simplify GitHub workflow to remove Gramine build steps --- .github/workflows/build-and-release.yml | 154 +++++++++--------------- 1 file changed, 57 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 506fbdb..650ad80 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -13,100 +13,60 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build Docker image - uses: docker/build-push-action@v4 - with: - context: . - load: true - tags: | - my-proof:${{ github.run_number }} - my-proof:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Clone and set up GSC - run: | - git clone https://github.com/gramineproject/gsc.git - cd gsc - python3 -m pip install --no-cache-dir 'docker>=7.1.0' 'jinja2>=3.1.4' 'tomli>=2.0.1' 'tomli-w>=1.0.0' 'pyyaml>=6.0.2' - - - name: Create signing key - run: | - echo "${{ secrets.SIGNING_KEY }}" > signing_key.pem - chmod 600 signing_key.pem - - - name: Build GSC image - run: | - cd gsc - ./gsc build my-proof ../my-proof.manifest.template -c ../config.yaml - - - name: Sign GSC image - run: | - cd gsc - ./gsc sign-image my-proof ../signing_key.pem -c ../config.yaml - - - name: Export GSC image to file - run: | - docker save gsc-my-proof:latest | gzip > gsc-my-proof-${{ github.run_number }}.tar.gz - - - name: Generate verification data - run: | - cd gsc - ./gsc info-image gsc-my-proof > ../sigstruct.txt - - - name: Upload image - uses: actions/upload-artifact@v3 - with: - name: gsc-my-proof-image - path: gsc-my-proof-${{ github.run_number }}.tar.gz - - - name: Upload verification data - uses: actions/upload-artifact@v3 - with: - name: gsc-my-proof-sigstruct - path: sigstruct.txt - - - name: Generate release body - run: | - echo "MRSIGNER: $(grep -oP 'mr_signer = "\K[^"]*' sigstruct.txt)" >> release_body.txt - echo "MRENCLAVE: $(grep -oP 'mr_enclave = "\K[^"]*' sigstruct.txt)" >> release_body.txt - echo "Image SHA256: $(sha256sum gsc-my-proof-${{ github.run_number }}.tar.gz | cut -d' ' -f1)" >> release_body.txt - - - name: Create Release and Upload Assets - uses: softprops/action-gh-release@v1 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ github.run_number }} - name: Release v${{ github.run_number }} - body_path: release_body.txt - draft: false - prerelease: false - files: | - ./gsc-my-proof-${{ github.run_number }}.tar.gz - ./sigstruct.txt - - - name: Cleanup signing key - if: always() - run: | - rm -f signing_key.pem - - - name: Log build result - if: always() - run: | - if [ ${{ job.status }} == "success" ]; then - echo "Build and release completed successfully" - else - echo "Build and release failed" - fi \ No newline at end of file + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Docker image + uses: docker/build-push-action@v4 + with: + context: . + load: true + tags: | + my-proof:${{ github.run_number }} + my-proof:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Export image to file + run: | + docker save my-proof:latest | gzip > my-proof-${{ github.run_number }}.tar.gz + + - name: Generate release body + run: | + echo "Image SHA256: $(sha256sum my-proof-${{ github.run_number }}.tar.gz | cut -d' ' -f1)" >> release_body.txt + + - name: Upload image + uses: actions/upload-artifact@v3 + with: + name: my-proof-image + path: my-proof-${{ github.run_number }}.tar.gz + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v1 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ github.run_number }} + name: Release v${{ github.run_number }} + body_path: release_body.txt + draft: false + prerelease: false + files: | + ./my-proof-${{ github.run_number }}.tar.gz + + - name: Log build result + if: always() + run: | + if [ ${{ job.status }} == "success" ]; then + echo "Build and release completed successfully" + else + echo "Build and release failed" + fi \ No newline at end of file From 932c87ce1274910d1cbd302316e02df3d39414bc Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 12:23:20 -0500 Subject: [PATCH 4/7] feat: remove sealed storage references and gramine-specific paths --- my_proof/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/my_proof/__main__.py b/my_proof/__main__.py index 3dba9e2..c3596d5 100644 --- a/my_proof/__main__.py +++ b/my_proof/__main__.py @@ -8,7 +8,7 @@ from my_proof.proof import Proof -INPUT_DIR, OUTPUT_DIR, SEALED_DIR = '/input', '/output', '/sealed' +INPUT_DIR, OUTPUT_DIR = '/input', '/output' logging.basicConfig(level=logging.INFO, format='%(message)s') @@ -17,7 +17,6 @@ def load_config() -> Dict[str, Any]: """Load proof configuration from environment variables.""" config = { 'dlp_id': 1234, # Set your own DLP ID here - 'use_sealing': os.path.isdir(SEALED_DIR), 'input_dir': INPUT_DIR, 'user_email': os.environ.get('USER_EMAIL', None), } From bf6a6d94d311adc9e73fda91fa5f4a44d0dcc720 Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 12:24:49 -0500 Subject: [PATCH 5/7] feat: update documentation to reflect TDX usage instead of gramine --- README.md | 103 ++++++++++++++---------------------------------------- 1 file changed, 27 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 6e790f6..3e76203 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,17 @@ This template provides a basic structure for building proof tasks that: } ``` -The project is designed to work with [Gramine](https://gramine.readthedocs.io/en/latest/), a lightweight library OS that enables running unmodified applications in secure enclaves, such as Intel SGX (Software Guard Extensions). This allows the code to run in a trusted execution environment, ensuring confidentiality and integrity of the computation. +The project is designed to work with Intel TDX (Trust Domain Extensions), providing hardware-level isolation and security guarantees for confidential computing workloads. ## Project Structure - `my_proof/`: Contains the main proof logic - - `proof.py`: Implements the proof generation logic - - `__main__.py`: Entry point for the proof execution + - `proof.py`: Implements the proof generation logic + - `__main__.py`: Entry point for the proof execution + - `models/`: Data models for the proof system - `demo/`: Contains sample input and output for testing -- `.github/workflows/`: CI/CD pipeline for building and releasing - `Dockerfile`: Defines the container image for the proof task -- `my-proof.manifest.template`: Gramine manifest template for running securely in an Intel SGX enclave -- `config.yaml`: Configuration file for Gramine Shielded Containers (GSC) +- `requirements.txt`: Python package dependencies ## Getting Started @@ -46,14 +45,14 @@ To use this template: 1. Fork this repository 2. Modify the `my_proof/proof.py` file to implement your specific proof logic -3. Update the `my-proof.manifest.template` if you need to add any additional files or change the configuration +3. Update the project dependencies in `requirements.txt` if needed 4. Commit your changes and push to your repository ## Customizing the Proof Logic The main proof logic is implemented in `my_proof/proof.py`. To customize it, update the `Proof.generate()` function to change how input files are processed. -The proof can be configured using environment variables. When running in an enclave, the environment variables must be defined in the `my-proof.manifest.template` file as well. The following environment variables are used for this demo proof: +The proof can be configured using environment variables: - `USER_EMAIL`: The email address of the data contributor, to verify data ownership @@ -61,78 +60,31 @@ If you want to use a language other than Python, you can modify the Dockerfile t ## Local Development -To run the proof locally, without Gramine, you can use Docker: +To run the proof locally for testing, you can use Docker: -``` +```bash docker build -t my-proof . docker run \ ---rm \ ---volume $(pwd)/demo/sealed:/sealed \ ---volume $(pwd)/demo/input:/input \ ---volume $(pwd)/demo/output:/output \ ---env USER_EMAIL=user123@gmail.com \ -my-proof + --rm \ + --volume $(pwd)/input:/input \ + --volume $(pwd)/output:/output \ + --env USER_EMAIL=user123@gmail.com \ + my-proof ``` -## Building and Releasing - -This template includes a GitHub Actions workflow that automatically: - -1. Builds a Docker image with your code -2. Creates a Gramine-shielded container (GSC) image -3. Publishes the GSC image as a GitHub release - -**Important:** To use this workflow, you must generate a signing key and add it to your GitHub secrets. Follow these steps: - -1. Generate a signing key (see instructions below) -2. Add the key as a GitHub secret named `SIGNING_KEY` -3. Push your changes to the `main` branch or create a pull request - -### Generating the Gramine Signing Key (Required) - -Before building and signing your graminized Docker image, you must generate a signing key. This key is crucial for creating secure SGX enclaves. Here's how to generate it: - -1. If you have Gramine installed: - - ``` - gramine-sgx-gen-private-key enclave-key.pem - ``` - -2. If you don't have Gramine, use OpenSSL: +## Running with Intel TDX - ``` - openssl genrsa -3 -out enclave-key.pem 3072 - ``` +Intel TDX (Trust Domain Extensions) provides hardware-based memory encryption and integrity protection for virtual machines. To run this container in a TDX-enabled environment, follow your infrastructure provider's specific instructions for deploying confidential containers. -After generating the key: +Common volume mounts and environment variables: -1. Keep this key secure, as it will be used to sign your enclaves. -2. Add the contents of `enclave-key.pem` as a GitHub secret named `SIGNING_KEY`. - -This key is essential for the `gsc sign-image` step in the GSC workflow. - -## Running with SGX - -Intel SGX (Software Guard Extensions) is a set of security-related instruction codes built into modern Intel CPUs. It allows parts of a program to be executed in a secure enclave, isolated from the rest of the system. - -To load a released image with docker, copy the URL from the release and run: - -``` -curl -L https://address/of/gsc-my-proof.tar.gz | docker load -``` - -To run the image: - -``` +```bash docker run \ ---rm \ ---volume /gsc-my-proof/input:/input \ ---volume /gsc-my-proof/output:/output \ ---device /dev/sgx_enclave:/dev/sgx_enclave \ ---volume /var/run/aesmd:/var/run/aesmd \ ---volume /mnt/gsc-my-proof/sealed:/sealed \ ---env USER_EMAIL=user123@gmail.com \ -gsc-my-proof + --rm \ + --volume /path/to/input:/input \ + --volume /path/to/output:/output \ + --env USER_EMAIL=user123@gmail.com \ + my-proof ``` Remember to populate the `/input` directory with the files you want to process. @@ -141,10 +93,9 @@ Remember to populate the `/input` directory with the files you want to process. This template leverages several security features: -1. **Secure Enclaves**: The proof runs inside an SGX enclave, isolating it from the rest of the system. -2. **Encrypted Storage**: The `/sealed` directory is automatically encrypted/decrypted by Gramine, providing secure storage for sensitive data. -3. **Input/Output Isolation**: Input and output directories are mounted separately, ensuring clear data flow boundaries. -4. **Minimal Attack Surface**: The Gramine manifest limits the files and resources accessible to the enclave, reducing potential vulnerabilities. +1. **Hardware-based Isolation**: The proof runs inside a TDX-protected environment, isolating it from the rest of the system +2. **Input/Output Isolation**: Input and output directories are mounted separately, ensuring clear data flow boundaries +3. **Minimal Container**: Uses a minimal Python base image to reduce attack surface ## Customization @@ -156,4 +107,4 @@ If you have suggestions for improving this template, please open an issue or sub ## License -[MIT License](LICENSE) +[MIT License](LICENSE) \ No newline at end of file From a974fb9a896dbb533bf88418a45a8a5f618956d5 Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 12:25:04 -0500 Subject: [PATCH 6/7] feat: update dependencies --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 572b352..76aa8db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pydantic +requests \ No newline at end of file From 8068441aade1af5b7b488e504d4a73016169e241 Mon Sep 17 00:00:00 2001 From: Volod Date: Fri, 29 Nov 2024 13:05:25 -0500 Subject: [PATCH 7/7] feat: remove manifest - no longer needed --- my-proof.manifest.template | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 my-proof.manifest.template diff --git a/my-proof.manifest.template b/my-proof.manifest.template deleted file mode 100644 index 0b2d7f8..0000000 --- a/my-proof.manifest.template +++ /dev/null @@ -1,31 +0,0 @@ -# Adjust this as needed. -sgx.enclave_size = "256M" - -# Increase this as needed, e.g., if you run a web server. -sgx.max_threads = 4 - -# Whitelist ENV variables that get passed to the enclave -# Using { passthrough = true } allows values to be passed in from the Satya node's /RunProof endpoint -loader.env.USER_EMAIL = { passthrough = true } - -# Gramine gives a warning that allowed_files is not safe in production, but it -# should generally be fine for our use case which inherently assumes that input -# files are untrusted until proven otherwise. -sgx.allowed_files = [ - "file:/input/", - "file:/output/", - # Required for internet access from inside the enclave - "file:/etc/hosts", - "file:/etc/resolv.conf", -] - -# These directories are mounted from the host, which will be a temporary directory from the Satya node that's running the proof. -fs.mounts = [ - { type = "encrypted", path = "/sealed", uri = "file:/sealed", key_name = "_sgx_mrenclave" }, - { path = "/input", uri = "file:/input" }, - { path = "/output", uri = "file:/output" }, -] - -# You can add other Gramine-manifest-compatible options as needed, see the -# Gramine documentation for more details: https://gramine.readthedocs.io. Note -# that gsc defines a number of manifest settings by default.