From cc3f12da97de9845c9d80ab4c57dcd04dcfca7c8 Mon Sep 17 00:00:00 2001 From: Eran Lerer Date: Thu, 16 Jan 2025 15:32:13 +0200 Subject: [PATCH 1/3] Creating local_state directory, and saving state to it (#1271) Signed-off-by: Lerer, Eran --- openfl-docker/gramine_app/fx.manifest.template | 1 + openfl-workspace/workspace/plan/defaults/aggregator.yaml | 2 +- openfl/component/aggregator/aggregator.py | 2 +- openfl/interface/interactive_api/experiment.py | 1 + openfl/interface/workspace.py | 2 ++ tests/openfl/federated/plan/plan_example.yaml | 1 + 6 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index 928dff0f56..55e20adf42 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -64,6 +64,7 @@ sgx.trusted_files = [ # One should be conservative as to which files are allowed, these can be modified by enclave. sgx.allowed_files = [ "file:{{ workspace_root }}/save", + "file:{{ workspace_root }}/local_state", "file:{{ workspace_root }}/logs", "file:{{ workspace_root }}/cert", "file:{{ workspace_root }}/data", diff --git a/openfl-workspace/workspace/plan/defaults/aggregator.yaml b/openfl-workspace/workspace/plan/defaults/aggregator.yaml index d4204e2b0c..9fc0481f29 100644 --- a/openfl-workspace/workspace/plan/defaults/aggregator.yaml +++ b/openfl-workspace/workspace/plan/defaults/aggregator.yaml @@ -2,4 +2,4 @@ template : openfl.component.Aggregator settings : db_store_rounds : 2 persist_checkpoint: True - persistent_db_path: save/tensor.db + persistent_db_path: local_state/tensor.db diff --git a/openfl/component/aggregator/aggregator.py b/openfl/component/aggregator/aggregator.py index 4b3ff2680c..c6829e75b9 100644 --- a/openfl/component/aggregator/aggregator.py +++ b/openfl/component/aggregator/aggregator.py @@ -1076,7 +1076,7 @@ def _compute_validation_related_task_metrics(self, task_name) -> dict: # FIXME: Configurable logic for min/max criteria in saving best. if "validate_agg" in tags: - # Compare the accuracy of the model, potentially save it + # Compare the accuracy of the model, potentially save it. if self.best_model_score is None or self.best_model_score < agg_results: logger.info( f"Round {round_number}: saved the best model with score {agg_results:f}" diff --git a/openfl/interface/interactive_api/experiment.py b/openfl/interface/interactive_api/experiment.py index cc3ffae3b2..0a9b383b2f 100644 --- a/openfl/interface/interactive_api/experiment.py +++ b/openfl/interface/interactive_api/experiment.py @@ -103,6 +103,7 @@ def _initialize_plan(self): # Create a folder to store plans os.makedirs("./plan", exist_ok=True) os.makedirs("./save", exist_ok=True) + os.makedirs("./local_state", exist_ok=True) # Load the default plan base_plan_path = WORKSPACE / "workspace/plan/plans/default/base_plan_interactive_api.yaml" plan = Plan.parse(base_plan_path, resolve=False) diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index dce6eb9dae..522ff99b5f 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -66,6 +66,7 @@ def create_dirs(prefix): (prefix / "data").mkdir(parents=True, exist_ok=True) # training data (prefix / "logs").mkdir(parents=True, exist_ok=True) # training logs (prefix / "save").mkdir(parents=True, exist_ok=True) # model weight saves / initialization + (prefix / "local_state").mkdir(parents=True, exist_ok=True) # persistent state (prefix / "src").mkdir(parents=True, exist_ok=True) # model code shutil.copyfile(WORKSPACE / "workspace" / ".workspace", prefix / ".workspace") @@ -354,6 +355,7 @@ def export_() -> str: # os.makedirs(os.path.join(tmp_dir, 'save'), exist_ok=True) os.makedirs(os.path.join(tmp_dir, "logs"), exist_ok=True) os.makedirs(os.path.join(tmp_dir, "data"), exist_ok=True) + os.makedirs(os.path.join(tmp_dir, "local_state"), exist_ok=True) shutil.copytree("src", os.path.join(tmp_dir, "src"), ignore=ignore) shutil.copytree("plan", os.path.join(tmp_dir, "plan"), ignore=ignore) shutil.copytree("save", os.path.join(tmp_dir, "save")) diff --git a/tests/openfl/federated/plan/plan_example.yaml b/tests/openfl/federated/plan/plan_example.yaml index af976f3f43..8afe672b11 100644 --- a/tests/openfl/federated/plan/plan_example.yaml +++ b/tests/openfl/federated/plan/plan_example.yaml @@ -9,6 +9,7 @@ aggregator : best_state_path : save/best.pbuf last_state_path : save/last.pbuf rounds_to_train : 10 + persistent_db_path: tensor.db collaborator : defaults : plan/defaults/collaborator.yaml From 810a4717d5a6c017a1ca00b28030ddae9ae65bf4 Mon Sep 17 00:00:00 2001 From: Nambi Srinivasan S Date: Fri, 17 Jan 2025 10:59:41 +0530 Subject: [PATCH 2/3] Bandit Scan as part of PR and precommit workflow (#1275) * Create bandit.yml 1) skips - will be added once we need block 2) severity set to HIGH * testing changes to exclude bandit in linter --------- Co-authored-by: Preethi --- .github/workflows/bandit.yml | 51 +++++++++++++++++++++ .pre-commit-config.yaml | 8 +++- README.md | 3 ++ pre_commit.toml | 7 +++ precommit-doc.md | 66 +++++++++++++++++++++++++++ precommit-setup.sh | 87 ++++++++++++++++++++++++++++++++++++ scripts/lint.sh | 2 +- 7 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/bandit.yml create mode 100644 pre_commit.toml create mode 100644 precommit-doc.md create mode 100644 precommit-setup.sh diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 0000000000..646bebbc61 --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,51 @@ +name: Bandit Code Scan + +on: + push: + branches: + - develop + - v1.7.x + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + bandit_scan: + if: github.event.pull_request.draft == false + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: Bandit Scan + runs-on: ubuntu-22.04 + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set Filename Suffix Report Date and Time + run: | + echo "REPORT_DATE=$(date +'%d-%b-%Y_%H-%M-%S')" >> $GITHUB_ENV + + - name: Define SARIF Report Path + run: echo "SARIF_REPORT_PATH=${{ github.workspace }}/results.sarif" >> $GITHUB_ENV + + - name: Perform Bandit Analysis + uses: PyCQA/bandit-action@v1 + with: + configfile: 'DEFAULT' + profile: 'DEFAULT' + tests: 'DEFAULT' + skips: 'DEFAULT' + severity: 'DEFAULT' + confidence: 'DEFAULT' + exclude: '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg' + baseline: 'DEFAULT' + ini: 'DEFAULT' + targets: '.' + + - name: Upload Bandit SARIF Report as Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: "bandit-report-summary_${{ env.REPORT_DATE }}" + path: ${{ env.SARIF_REPORT_PATH }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb9b3a0916..e2008b7ec5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,4 +27,10 @@ repos: - --in-place - --remove-unused-variables - --recursive - - --ignore-pass-statements \ No newline at end of file + - --ignore-pass-statements + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + args: ["-c", "pre_commit.toml"] + additional_dependencies: ["bandit[toml]"] \ No newline at end of file diff --git a/README.md b/README.md index 5ed611dd72..275899ed8b 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,9 @@ OpenFL supports popular aggregation algorithms out-of-the-box, with more algorit | FedProx | [Li et al., 2020](https://arxiv.org/pdf/1812.06127.pdf) | yes | yes | - | | FedCurv | [Shoham et al., 2019](https://arxiv.org/pdf/1910.07796.pdf) | yes | - | - | +### Enabling Bandit Precommit +To ensure that precommit is setup in your local for Bandit Scan. For more details, kindly follow this doc: [Setup Guide - Precommit](precommit-doc.md) + ## Contributing We welcome contributions! Please refer to the [contributing guidelines](https://openfl.readthedocs.io/en/latest/contributing.html). diff --git a/pre_commit.toml b/pre_commit.toml new file mode 100644 index 0000000000..6a748a6b02 --- /dev/null +++ b/pre_commit.toml @@ -0,0 +1,7 @@ +[tool.bandit] +# Exclude specific directories or files from the scan +# exclude = ["tests/", "docs/"] + +# Set the severity and confidence levels +severity = "HIGH" +confidence = "HIGH" diff --git a/precommit-doc.md b/precommit-doc.md new file mode 100644 index 0000000000..9782a5d01e --- /dev/null +++ b/precommit-doc.md @@ -0,0 +1,66 @@ +## Pre-commit with Bandit + +To ensure code quality and security, we use [pre-commit](https://pre-commit.com/) with [Bandit](https://bandit.readthedocs.io/en/latest/) to automatically scan for security issues before commits. + +Follow the steps below to set up and use pre-commit in your local development environment. + +### Setup + +1. **Clone the repository**: + + ```sh + git clone https://github.com/intel-innersource/frameworks.ai.openfl.openfl-security.git + cd frameworks.ai.openfl.openfl-security + ``` + +2. **Run the setup script**: + + We have provided a `precommit-setup.sh` script to simplify the installation process. This script will install pre-commit and set up the pre-commit hooks. + + ```sh + ./precommit-setup.sh + ``` + + The `setup.sh` script performs the following actions: + - Check for prerequisties in local: (python, pip) + - Installs pre-commit if it is not already installed. + - Installs the pre-commit hooks defined in the .pre-commit-config.yaml file. + +3. **Verify the installation**: + + After running the setup script, you can verify that pre-commit is installed and the hooks are set up correctly by running: + + ```sh + pre-commit --version + pre-commit install + ``` + +### Usage + +Once the pre-commit hooks are installed, Bandit scans will automatically run before each commit. If any issues are found, the commit will be aborted, and you will need to fix the issues before committing again. + +1. **Make changes to your code**: + + Edit your files as needed. + +2. **Stage your changes**: + + ```sh + git add + ``` + +3. **Commit your changes**: + + ```sh + git commit -m "Your commit message" + ``` + + During the commit process, pre-commit will automatically run the Bandit scan. If the scan is successful, the commit will proceed. If any issues are found, the commit will be aborted, and you will need to address the issues before committing again. + +### How to bypass precommit hooks? + +To exclude the bandit pre-commit hook when making a Git commit, you can use the --no-verify option. This bypasses any pre-commit hooks that are set up in your repository. + +```sh +git commit --no-verify -m "Your commit message" +``` diff --git a/precommit-setup.sh b/precommit-setup.sh new file mode 100644 index 0000000000..9b55289bc9 --- /dev/null +++ b/precommit-setup.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# Function to add the installation path to PATH +add_to_path() { + if [[ ":$PATH:" != *":$1:"* ]]; then + export PATH="$PATH:$1" + echo "Added $1 to PATH" + else + echo "$1 is already in PATH" + fi +} + +# Function to check if Python and pip are installed +check_python_and_pip() { + if ! command -v python3 &> /dev/null; then + echo "Python3 is not installed. Please install Python3 and try again." + exit 1 + fi + + if ! command -v pip &> /dev/null; then + echo "pip is not installed. Please install pip and try again." + exit 1 + fi +} + +# Function to install pre-commit +install_precommit() { + if ! command -v pre-commit &> /dev/null; then + echo "pre-commit not found, installing..." + pip install --user pre-commit + else + echo "pre-commit is already installed" + fi +} + +# Check if Python and pip are installed +check_python_and_pip + +# Detect the operating system +OS="$(uname -s)" +case "$OS" in + Linux*) + echo "Detected Linux" + INSTALL_PATH="$HOME/.local/bin" + install_precommit + add_to_path "$INSTALL_PATH" + ;; + Darwin*) + echo "Detected MacOS" + INSTALL_PATH="$HOME/.local/bin" + install_precommit + add_to_path "$INSTALL_PATH" + ;; + CYGWIN*|MINGW32*|MSYS*|MINGW*) + echo "Detected Windows" + INSTALL_PATH="$HOME/AppData/Local/Packages/PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0/LocalCache/local-packages/Python312/Scripts" + install_precommit + add_to_path "$INSTALL_PATH" + ;; + *) + echo "Unknown OS" + exit 1 + ;; +esac + +# Add the installation path to the shell profile for persistence +if [[ "$OS" == "Linux" || "$OS" == "Darwin" ]]; then + SHELL_PROFILE="$HOME/.bashrc" + if [[ -f "$HOME/.zshrc" ]]; then + SHELL_PROFILE="$HOME/.zshrc" + fi + echo "export PATH=\$PATH:$INSTALL_PATH" >> "$SHELL_PROFILE" + source "$SHELL_PROFILE" +elif [[ "$OS" == "CYGWIN"* || "$OS" == "MINGW"* || "$OS" == "MSYS"* ]]; then + SHELL_PROFILE="$HOME/.bash_profile" + echo "export PATH=\$PATH:$INSTALL_PATH" >> "$SHELL_PROFILE" + source "$SHELL_PROFILE" +fi + +# Verify the installation +if command -v pre-commit &> /dev/null; then + echo "pre-commit installation successful" + pre-commit --version +else + echo "pre-commit installation failed" + exit 1 +fi diff --git a/scripts/lint.sh b/scripts/lint.sh index ac0b97e0a5..98261135bc 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -4,7 +4,7 @@ set -Eeuo pipefail base_dir=$(dirname $(dirname $0)) # Run the pre-commit checks -pre-commit run --all-files +SKIP=bandit pre-commit run --all-files ruff check --config "${base_dir}/pyproject.toml" openfl/ From 3751f704cbfb77aa9deaeaa934ccffa228553958 Mon Sep 17 00:00:00 2001 From: rajithkrishnegowda <134698520+rajithkrishnegowda@users.noreply.github.com> Date: Fri, 17 Jan 2025 13:43:27 +0530 Subject: [PATCH 3/3] adding steps to run with enhanced security (#1281) * adding steps to run with enhanced security * added changes --- openfl-docker/README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/openfl-docker/README.md b/openfl-docker/README.md index da8540770d..73f15a5419 100644 --- a/openfl-docker/README.md +++ b/openfl-docker/README.md @@ -86,4 +86,28 @@ docker run --rm \ -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ --mount type=bind,source=./certs.tar,target=/certs.tar \ example_workspace bash -c "gramine-sgx fx collaborator start ..." -``` \ No newline at end of file +``` + +### Running OpenFL Container in Production +For running [TaskRunner API](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#running-the-task-runner) in a production environment with enhanced security, use the following parameters to limit CPU, memory, and process IDs, and to prevent privilege escalation: + +**Example Command**: +```shell +docker run --rm --name --network openfl \ + -v $WORKING_DIRECTORY:/workdir-openfl \ + --cpus="0.1" \ + --memory="512m" \ + --pids-limit 100 \ + --security-opt no-new-privileges \ + openfl:latest +``` +**Parameters**: +```shell +--cpus="0.1": Limits the container to 10% of a single CPU core. +--memory="512m": Limits the container to 512MB of memory. +--pids-limit 100: Limits the number of processes to 100. +--security-opt no-new-privileges: Prevents the container from gaining additional privileges. +``` +These settings help ensure that your containerized application runs securely and efficiently in a production environment + +**Note**: The numbers suggested here are examples/minimal suggestions and need to be adjusted according to the environment and the type of experiments you are aiming to run. \ No newline at end of file