diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml new file mode 100644 index 0000000000..d8f7480ea1 --- /dev/null +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -0,0 +1,97 @@ +# Tests an FL experiment in a Dockerized environment. +name: TaskRunner (docker/gramine-direct) + +on: + pull_request: + branches: [ develop ] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +jobs: + build: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install . + + - name: Create workspace image + run: | + fx workspace create --prefix example_workspace --template keras_cnn_mnist + cd example_workspace + fx plan initialize -a localhost + fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} + + - name: Create certificate authority for workspace + run: | + cd example_workspace + fx workspace certify + + - name: Create signed cert for collaborator + run: | + cd example_workspace + fx collaborator create -d 1 -n charlie --silent + fx collaborator generate-cert-request -n charlie --silent + fx collaborator certify --request-pkg col_charlie_to_agg_cert_request.zip --silent + + # Pack the collaborator's private key, signed cert, and data.yaml into a tarball + tarfiles="plan/data.yaml agg_to_col_charlie_signed_cert.zip" + for entry in cert/client/*; do + if [[ "$entry" == *.key ]]; then + tarfiles="$tarfiles $entry" + fi + done + + tar -cf cert_col_charlie.tar $tarfiles + + # Clean up + rm -f $tarfiles + rm -f col_charlie_to_agg_cert_request.zip + + - name: Create signed cert for aggregator + run: | + cd example_workspace + fx aggregator generate-cert-request --fqdn localhost + fx aggregator certify --fqdn localhost --silent + + # Pack all files that aggregator needs to start training + tar -cf cert_agg.tar plan cert save + + # Remove the directories after archiving + rm -rf plan cert save + + - name: Load workspace image + run: | + cd example_workspace + docker load -i example_workspace.tar + + - name: Run aggregator and collaborator + run: | + cd example_workspace + + set -x + docker run --rm \ + --network host \ + --security-opt seccomp=unconfined \ + --mount type=bind,source=./cert_agg.tar,target=/certs.tar \ + --env KERAS_HOME=/tmp \ + example_workspace bash -c "tar -xf /certs.tar && gramine-direct fx aggregator start" & + + # TODO: Run with two collaborators instead. + docker run --rm \ + --network host \ + --security-opt seccomp=unconfined \ + --mount type=bind,source=./cert_col_charlie.tar,target=/certs.tar \ + --env KERAS_HOME=/tmp \ + example_workspace bash -c "tar -xf /certs.tar && fx collaborator certify --import agg_to_col_charlie_signed_cert.zip && gramine-direct fx collaborator start -n charlie" \ No newline at end of file diff --git a/.github/workflows/dockerization.yml b/.github/workflows/tr_docker_native.yml similarity index 98% rename from .github/workflows/dockerization.yml rename to .github/workflows/tr_docker_native.yml index 81d29f5f2a..899fcd8296 100644 --- a/.github/workflows/dockerization.yml +++ b/.github/workflows/tr_docker_native.yml @@ -1,5 +1,5 @@ # Tests an FL experiment in a Dockerized environment. -name: Dockerization +name: TaskRunner (docker/native) on: pull_request: diff --git a/openfl-docker/Dockerfile.base b/openfl-docker/Dockerfile.base index 0b5d746aca..f58d83747f 100644 --- a/openfl-docker/Dockerfile.base +++ b/openfl-docker/Dockerfile.base @@ -1,7 +1,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # ------------------------------------ -# OpenFL Base Image +# OpenFL Base Image w/ Gramine support # $> docker build . -t openfl -f Dockerfile.base [--build-arg OPENFL_REVISION=GIT_URL@COMMIT_ID] # ------------------------------------ FROM ubuntu:22.04 AS base @@ -15,25 +15,41 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ apt-get update && \ apt-get install -y \ git \ + curl \ python3-pip \ python3.10-dev \ + python3.10-venv \ ca-certificates \ build-essential \ --no-install-recommends && \ apt-get purge -y linux-libc-dev && \ rm -rf /var/lib/apt/lists/* +# Create a python virtual environment. +RUN python3.10 -m venv /opt/venv && \ + /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel +ENV PATH=/opt/venv/bin:$PATH + +# Install Gramine +RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ + curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ jammy main" \ + | tee /etc/apt/sources.list.d/gramine.list && \ + curl -fsSLo /usr/share/keyrings/intel-sgx-deb.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx-deb.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" \ + | tee /etc/apt/sources.list.d/intel-sgx.list && \ + apt-get update && \ + apt-get install -y gramine --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# Install OpenFL. +ARG OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@v1.6 +RUN pip install --no-cache-dir git+${OPENFL_REVISION} && \ + INSTALL_SOURCES=yes /opt/venv/lib/python3.10/site-packages/openfl-docker/licenses.sh + # Create an unprivileged user. RUN groupadd -g 1001 default && \ useradd -m -u 1001 -g default user USER user -WORKDIR /home/user -ENV PATH=/home/user/.local/bin:$PATH - -# Install OpenFL. -ARG OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@v1.6 -RUN pip install --no-cache-dir -U pip setuptools wheel && \ - pip install --no-cache-dir git+${OPENFL_REVISION} && \ - INSTALL_SOURCES=yes /home/user/.local/lib/python3.10/site-packages/openfl-docker/licenses.sh CMD ["/bin/bash"] diff --git a/openfl-docker/Dockerfile.workspace b/openfl-docker/Dockerfile.workspace index 08446663c6..0165f557c1 100644 --- a/openfl-docker/Dockerfile.workspace +++ b/openfl-docker/Dockerfile.workspace @@ -1,18 +1,33 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # ------------------------------------ -# Workspace Image +# Gramine-ready Workspace Image +# Usage: +# $> docker build . -t openfl-workspace -f Dockerfile.workspace \ +# [--build-arg BASE_IMAGE=openfl:latest] \ +# [--build-arg WORKSPACE_NAME=WORKSPACE_NAME] \ +# [--secret id=signer-key,src=signer-key.pem] # ------------------------------------ ARG BASE_IMAGE=openfl:latest FROM ${BASE_IMAGE} +USER root SHELL ["/bin/bash", "-o", "pipefail", "-c"] -USER user +# Import workspace +WORKDIR / ARG WORKSPACE_NAME -COPY ${WORKSPACE_NAME}.zip . -RUN fx workspace import --archive ${WORKSPACE_NAME}.zip && \ - pip install --no-cache-dir -r ${WORKSPACE_NAME}/requirements.txt +COPY ${WORKSPACE_NAME}.zip /workspace.zip +RUN fx workspace import --archive /workspace.zip && \ + pip install --no-cache-dir -r /workspace/requirements.txt + +# Build enclaves +WORKDIR /workspace +RUN --mount=type=secret,id=signer-key,dst=/key.pem \ + cp -r /opt/venv/lib/python3.10/site-packages/openfl-docker/gramine_app/* /workspace/ && \ + make SGX=1 SGX_SIGNER_KEY=/key.pem >> fx.mr_enclave && \ + echo "$(cat fx.mr_enclave)" && \ + chown -R user /workspace -WORKDIR /home/user/${WORKSPACE_NAME} +USER user CMD ["/bin/bash"] \ No newline at end of file diff --git a/openfl-docker/README.md b/openfl-docker/README.md new file mode 100644 index 0000000000..da8540770d --- /dev/null +++ b/openfl-docker/README.md @@ -0,0 +1,89 @@ +# Using OpenFL within a Container + +OpenFL can be used within a container for simulating Federated Learning experiments, or to deploy real-world experiments within Trusted Execution Environments (TEEs). + +## Base Image + +To develop or simulate experiments within a container, build the base image (or pull one from docker hub). + +```shell +# Pull latest stable base image +$> docker pull intel/openfl + +# Or, build a base image from the latest source code +$> docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@develop +``` + +Run the container: +```shell +user@vm:~/openfl$ docker run -it --rm openfl:latest bash +user@7b40624c207a:/$ fx +OpenFL - Open Federated Learning + +BASH COMPLETE ACTIVATION + +Run in terminal: + _FX_COMPLETE=bash_source fx > ~/.fx-autocomplete.sh + source ~/.fx-autocomplete.sh +If ~/.fx-autocomplete.sh has already exist: + source ~/.fx-autocomplete.sh + +CORRECT USAGE + +fx [options] [command] [subcommand] [args] +``` + +## Deployment +This section assumes familiarity with the [TaskRunner API](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#running-the-task-runner). + +### Building a workspace image +OpenFL supports [Gramine-based](https://gramine.readthedocs.io/en/stable/) TEEs that run within SGX. + +To build a TEE-ready workspace image, run the following command from an existing workspace directory. Ensure PKI setup and plan confirmations are done before this step. + +```shell +# Optional, generate an enclave signing key (auto-generated otherwise) +user@vm:~/example_workspace$ openssl genrsa -out key.pem -3 3072 +user@vm:~/example_workspace$ fx workspace dockerize --enclave-key ./key.pem --save +``` +This command builds the base image and a TEE-ready workspace image. Refer to `fx workspace dockerize --help` for more details. + +A signed docker image named `example_workspace.tar` will be saved in the workspace. This image (along with respective PKI certificates) can be shared across participating entities. + +### Running without a TEE +Using native `fx` command within the image will run the experiment without TEEs. + +```shell +# Aggregator +docker run --rm \ + --network host \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "fx aggregator start ..." + +# Collaborator(s) +docker run --rm \ + --network host \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "fx collaborator start ..." +``` + +### Running within a TEE +To run `fx` within a TEE, mount SGX device and AESMD volumes. In addition, prefix the `fx` command with `gramine-sgx` directive. +```shell +# Aggregator +docker run --rm \ + --network host \ + --device=/dev/sgx_enclave \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "gramine-sgx fx aggregator start ..." + +# Collaborator(s) +docker run --rm \ + --network host \ + --device=/dev/sgx_enclave \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "gramine-sgx fx collaborator start ..." +``` \ No newline at end of file diff --git a/openfl-docker/gramine_app/Makefile b/openfl-docker/gramine_app/Makefile new file mode 100644 index 0000000000..4dbc8ef142 --- /dev/null +++ b/openfl-docker/gramine_app/Makefile @@ -0,0 +1,54 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# ------------------------------------ +# Makefile for Gramine application within a container +# Usage: +# 1. Activate the python venv. +# 2. Provide paths VENV_ROOT and WORKSPACE_ROOT. +# 3. make SGX=0/1 [SGX_SIGNER_KEY=] +# ------------------------------------ +VENV_ROOT ?= $(shell dirname $(shell dirname $(shell which python))) +WORKSPACE_ROOT ?= $(shell pwd) +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) +SGX_SIGNER_KEY ?= /key.pem + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +.PHONY: all +all: fx.manifest +ifeq ($(SGX),1) +all: fx.manifest.sgx fx.sig +endif + +fx.manifest: fx.manifest.template + @echo "Making fx.manifest file" + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dvenv_root=$(VENV_ROOT) \ + -Dentrypoint=$(VENV_ROOT)/bin/fx \ + -Dworkspace_root=$(WORKSPACE_ROOT) \ + $< >$@ + +fx.manifest.sgx: fx.manifest + @echo "Making fx.manifest.sgx file" + @test -s $(SGX_SIGNER_KEY) || \ + { echo "SGX signer private key was not found, please specify SGX_SIGNER_KEY!"; exit 1; } + @gramine-sgx-sign \ + --key $(SGX_SIGNER_KEY) \ + --manifest $< \ + --output $@ | tail -n 1 | tr -d ' ' | xargs -I {} echo "fx.mr_enclave={}" + +fx.sig: fx.manifest.sgx + +.PHONY: clean +clean: + $(RM) *.manifest *.manifest.sgx *.token *.sig OUTPUT* *.PID TEST_STDOUT TEST_STDERR + $(RM) -r scripts/__pycache__ + +.PHONY: distclean +distclean: clean diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template new file mode 100755 index 0000000000..928dff0f56 --- /dev/null +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -0,0 +1,73 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# ------------------------------------- +# Enclave Manifest for OpenFL TaskRunner API. +# This defines the configuration for the Gramine loader to run a Python application. +# ------------------------------------- + +libos.entrypoint = "{{ entrypoint }}" +loader.entrypoint.uri = "file:{{ gramine.libos }}" +loader.log_level = "{{ log_level }}" + +loader.insecure__use_cmdline_argv = true +loader.insecure__use_host_env = true + +loader.env.LD_LIBRARY_PATH = "{{ venv_root }}:{{ arch_libdir }}:/usr/{{ arch_libdir }}:/lib:/usr/lib" +loader.env.SSL_CERT_DIR = "/etc/ssl/certs" + +# Filesystem configuration within Gramine LibOS +fs.start_dir = "{{ workspace_root }}" +fs.mounts = [ + # System mounts (URI: path on host, PATH: pointer inside gramine) + { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" }, + { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" }, + { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, + { uri = "file:/usr", path = "/usr" }, + { type = "tmpfs", path = "/tmp" }, + # User-defined mounts specific to the application. + { uri = "file:{{ workspace_root }}", path = "{{ workspace_root }}" }, + { uri = "file:{{ venv_root }}", path = "{{ venv_root }}" }, +] + +# System configuration +sys.stack.size = "4M" +sys.brk.max_size = "1M" +sys.enable_sigterm_injection = true +sys.enable_extra_runtime_domain_names_conf = true + +# SGX configuration +sgx.debug = false +sgx.enclave_size = "16G" +sgx.preheat_enclave = false +sgx.remote_attestation = "dcap" +sgx.max_threads = 512 + +# List of trusted files, that are hashed and signed by the enclave. +# If these files change after signing of an enclave, application cannot run. +sgx.trusted_files = [ + "file:{{ gramine.libos }}", + "file:{{ entrypoint }}", + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", + "file:/usr/{{ arch_libdir }}/", + "file:/etc/ssl/certs/", + "file:{{ python.stdlib }}/", + "file:{{ python.distlib }}/", +{% for path in python.get_sys_path('python') %} + "file:{{ path }}{{ '/' if path.is_dir() else '' }}", +{% endfor %} + "file:{{ venv_root }}/", + "file:{{ workspace_root }}/src/", +] + +# List of allowed files that SGX enclave does NOT verify with signatures. +# One should be conservative as to which files are allowed, these can be modified by enclave. +sgx.allowed_files = [ + "file:{{ workspace_root }}/save", + "file:{{ workspace_root }}/logs", + "file:{{ workspace_root }}/cert", + "file:{{ workspace_root }}/data", + "file:{{ workspace_root }}/plan/cols.yaml", + "file:{{ workspace_root }}/plan/data.yaml", + "file:{{ workspace_root }}/plan/plan.yaml", +] diff --git a/openfl-docker/licenses.sh b/openfl-docker/licenses.sh index 005a8bccfa..4eba1e8c97 100755 --- a/openfl-docker/licenses.sh +++ b/openfl-docker/licenses.sh @@ -45,6 +45,6 @@ if [ "$INSTALL_SOURCES" = "yes" ]; then # Append dependency list to all_dependencies.txt pip-licenses | awk '{for(i=1;i<=NF;i++) if(i!=2) printf $i" "; print ""}' | tee -a all_dependencies.txt - # Download source packages for Python packages with specific licenses - pip-licenses | grep -E 'GPL|MPL|EPL' | awk '{OFS="=="} {print $1,$2}' | xargs pip download --no-binary :all: + # Download source packages for Python packages (if exists) with specific licenses + pip-licenses | grep -E 'GPL|MPL|EPL' | awk '{OFS="=="} {print $1,$2}' | xargs -I {} sh -c 'pip download --no-binary :all: {} || true' fi diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 260c71eed1..b138ad67db 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -389,6 +389,17 @@ def export_() -> str: default=False, help="If set, rebuilds docker images with `--no-cache` option.", ) +@option( + "--enclave-key", + "enclave_key", + type=str, + required=False, + help=( + "Path to an enclave signing key. If not provided, a key will be auto-generated in the workspace. " + "Note that this command builds a TEE-ready image, key is NOT packaged along with the image. " + "You have the flexibility to not run inside a TEE later." + ), +) @option( "--revision", required=False, @@ -401,8 +412,8 @@ def export_() -> str: ), ) @pass_context -def dockerize_(context, save, rebuild, revision): - """Package current workspace as a Docker image.""" +def dockerize_(context, save: bool, rebuild: bool, enclave_key: str, revision: str): + """Package current workspace as a TEE-ready Docker image.""" # Docker build options options = [] @@ -430,10 +441,24 @@ def dockerize_(context, save, rebuild, revision): _execute(base_image_build_cmd) # Build workspace image. + options = [] + options.append("--no-cache" if rebuild else "") + options = " ".join(options) + if enclave_key is None: + _execute("openssl genrsa -out key.pem -3 3072") + enclave_key = os.path.abspath("key.pem") + logging.info(f"Generated new enclave key: {enclave_key}") + else: + enclave_key = os.path.abspath(enclave_key) + if not os.path.exists(enclave_key): + raise FileNotFoundError(f"Enclave key `{enclave_key}` does not exist") + logging.info(f"Using enclave key: {enclave_key}") + logging.info("Building workspace image") ws_image_build_cmd = ( "DOCKER_BUILDKIT=1 docker build {options} " "--build-arg WORKSPACE_NAME={workspace_name} " + "--secret id=signer-key,src={enclave_key} " "-t {image_name} " "-f {dockerfile} " "{build_context}" @@ -441,6 +466,7 @@ def dockerize_(context, save, rebuild, revision): options=options, image_name=workspace_name, workspace_name=workspace_name, + enclave_key=enclave_key, dockerfile=os.path.join(SITEPACKS, "openfl-docker", "Dockerfile.workspace"), build_context=".", )