Merge branch 'develop' into dependabot/pip/ruff-0.9.2

.github/workflows/bandit.yml

@@ -0,0 +1,51 @@
+name: Bandit Code Scan
+
+on:
+  push:
+    branches: 
+      - develop
+      - v1.7.x
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  bandit_scan:
+    if: github.event.pull_request.draft == false
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: Bandit Scan
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set Filename Suffix Report Date and Time
+        run: |
+          echo "REPORT_DATE=$(date +'%d-%b-%Y_%H-%M-%S')" >> $GITHUB_ENV
+    
+      - name: Define SARIF Report Path
+        run: echo "SARIF_REPORT_PATH=${{ github.workspace }}/results.sarif" >> $GITHUB_ENV
+
+      - name: Perform Bandit Analysis
+        uses: PyCQA/bandit-action@v1
+        with:
+          configfile: 'DEFAULT'
+          profile: 'DEFAULT'
+          tests: 'DEFAULT'
+          skips: 'DEFAULT'
+          severity: 'DEFAULT'
+          confidence: 'DEFAULT'
+          exclude: '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg'
+          baseline: 'DEFAULT'
+          ini: 'DEFAULT'
+          targets: '.'
+
+      - name: Upload Bandit SARIF Report as Artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "bandit-report-summary_${{ env.REPORT_DATE }}"
+          path: ${{ env.SARIF_REPORT_PATH }}

.pre-commit-config.yaml

@@ -27,4 +27,10 @@ repos:
           - --in-place
           - --remove-unused-variables
           - --recursive
-          - --ignore-pass-statements
+          - --ignore-pass-statements
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.4
+    hooks:
+      - id: bandit
+        args: ["-c", "pre_commit.toml"]
+        additional_dependencies: ["bandit[toml]"]

README.md

@@ -71,6 +71,9 @@ OpenFL supports popular aggregation algorithms out-of-the-box, with more algorit
 | FedProx | [Li et al., 2020](https://arxiv.org/pdf/1812.06127.pdf) | yes | yes | - |
 | FedCurv | [Shoham et al., 2019](https://arxiv.org/pdf/1910.07796.pdf) | yes | - | - |
 
+### Enabling Bandit Precommit
+To ensure that precommit is setup in your local for Bandit Scan. For more details, kindly follow this doc: [Setup Guide - Precommit](precommit-doc.md)
+
 ## Contributing
 We welcome contributions! Please refer to the [contributing guidelines](https://openfl.readthedocs.io/en/latest/contributing.html).
 

openfl-docker/README.md

@@ -86,4 +86,28 @@ docker run --rm \
   -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \
   --mount type=bind,source=./certs.tar,target=/certs.tar \
   example_workspace bash -c "gramine-sgx fx collaborator start ..."
-```
+```
+
+### Running OpenFL Container in Production
+For running [TaskRunner API](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#running-the-task-runner) in a production environment with enhanced security, use the following parameters to limit CPU, memory, and process IDs, and to prevent privilege escalation:
+
+**Example Command**:
+```shell
+docker run --rm --name <Aggregator/Collaborator> --network openfl \
+  -v $WORKING_DIRECTORY:/workdir-openfl \
+  --cpus="0.1" \
+  --memory="512m" \
+  --pids-limit 100 \
+  --security-opt no-new-privileges \
+  openfl:latest
+```
+**Parameters**:
+```shell
+--cpus="0.1": Limits the container to 10% of a single CPU core.
+--memory="512m": Limits the container to 512MB of memory.
+--pids-limit 100: Limits the number of processes to 100.
+--security-opt no-new-privileges: Prevents the container from gaining additional privileges.
+```
+These settings help ensure that your containerized application runs securely and efficiently in a production environment
+
+**Note**: The numbers suggested here are examples/minimal suggestions and need to be adjusted according to the environment and the type of experiments you are aiming to run.

openfl-docker/gramine_app/fx.manifest.template

@@ -64,6 +64,7 @@ sgx.trusted_files = [
 # One should be conservative as to which files are allowed, these can be modified by enclave.
 sgx.allowed_files = [
   "file:{{ workspace_root }}/save",
+  "file:{{ workspace_root }}/local_state",
   "file:{{ workspace_root }}/logs",
   "file:{{ workspace_root }}/cert",
   "file:{{ workspace_root }}/data",

openfl-workspace/workspace/plan/defaults/aggregator.yaml

@@ -2,4 +2,4 @@ template : openfl.component.Aggregator
 settings :
     db_store_rounds : 2
     persist_checkpoint: True
-    persistent_db_path: save/tensor.db
+    persistent_db_path: local_state/tensor.db

openfl/component/aggregator/aggregator.py

@@ -1076,7 +1076,7 @@ def _compute_validation_related_task_metrics(self, task_name) -> dict:
 
                 # FIXME: Configurable logic for min/max criteria in saving best.
                 if "validate_agg" in tags:
-                    # Compare the accuracy of the model, potentially save it
+                    # Compare the accuracy of the model, potentially save it.
                     if self.best_model_score is None or self.best_model_score < agg_results:
                         logger.info(
                             f"Round {round_number}: saved the best model with score {agg_results:f}"

openfl/interface/interactive_api/experiment.py

@@ -103,6 +103,7 @@ def _initialize_plan(self):
         # Create a folder to store plans
         os.makedirs("./plan", exist_ok=True)
         os.makedirs("./save", exist_ok=True)
+        os.makedirs("./local_state", exist_ok=True)
         # Load the default plan
         base_plan_path = WORKSPACE / "workspace/plan/plans/default/base_plan_interactive_api.yaml"
         plan = Plan.parse(base_plan_path, resolve=False)

openfl/interface/workspace.py

@@ -66,6 +66,7 @@ def create_dirs(prefix):
     (prefix / "data").mkdir(parents=True, exist_ok=True)  # training data
     (prefix / "logs").mkdir(parents=True, exist_ok=True)  # training logs
     (prefix / "save").mkdir(parents=True, exist_ok=True)  # model weight saves / initialization
+    (prefix / "local_state").mkdir(parents=True, exist_ok=True)  # persistent state
     (prefix / "src").mkdir(parents=True, exist_ok=True)  # model code
 
     shutil.copyfile(WORKSPACE / "workspace" / ".workspace", prefix / ".workspace")
@@ -354,6 +355,7 @@ def export_() -> str:
     # os.makedirs(os.path.join(tmp_dir, 'save'), exist_ok=True)
     os.makedirs(os.path.join(tmp_dir, "logs"), exist_ok=True)
     os.makedirs(os.path.join(tmp_dir, "data"), exist_ok=True)
+    os.makedirs(os.path.join(tmp_dir, "local_state"), exist_ok=True)
     shutil.copytree("src", os.path.join(tmp_dir, "src"), ignore=ignore)
     shutil.copytree("plan", os.path.join(tmp_dir, "plan"), ignore=ignore)
     shutil.copytree("save", os.path.join(tmp_dir, "save"))

pre_commit.toml

@@ -0,0 +1,7 @@
+[tool.bandit]
+# Exclude specific directories or files from the scan
+# exclude = ["tests/", "docs/"]
+
+# Set the severity and confidence levels
+severity = "HIGH"
+confidence = "HIGH"

precommit-doc.md

@@ -0,0 +1,66 @@
+## Pre-commit with Bandit
+
+To ensure code quality and security, we use [pre-commit](https://pre-commit.com/) with [Bandit](https://bandit.readthedocs.io/en/latest/) to automatically scan for security issues before commits. 
+
+Follow the steps below to set up and use pre-commit in your local development environment.
+
+### Setup
+
+1. **Clone the repository**:
+
+   ```sh
+   git clone https://github.com/intel-innersource/frameworks.ai.openfl.openfl-security.git
+   cd frameworks.ai.openfl.openfl-security
+   ```
+
+2. **Run the setup script**:
+
+   We have provided a `precommit-setup.sh` script to simplify the installation process. This script will install pre-commit and set up the pre-commit hooks.
+
+   ```sh
+   ./precommit-setup.sh
+   ```
+
+   The `setup.sh` script performs the following actions:
+   - Check for prerequisties in local: (python, pip)
+   - Installs pre-commit if it is not already installed.
+   - Installs the pre-commit hooks defined in the .pre-commit-config.yaml file.
+
+3. **Verify the installation**:
+
+   After running the setup script, you can verify that pre-commit is installed and the hooks are set up correctly by running:
+
+   ```sh
+   pre-commit --version
+   pre-commit install
+   ```
+
+### Usage
+
+Once the pre-commit hooks are installed, Bandit scans will automatically run before each commit. If any issues are found, the commit will be aborted, and you will need to fix the issues before committing again.
+
+1. **Make changes to your code**:
+
+   Edit your files as needed.
+
+2. **Stage your changes**:
+
+   ```sh
+   git add <file>
+   ```
+
+3. **Commit your changes**:
+
+   ```sh
+   git commit -m "Your commit message"
+   ```
+
+   During the commit process, pre-commit will automatically run the Bandit scan. If the scan is successful, the commit will proceed. If any issues are found, the commit will be aborted, and you will need to address the issues before committing again.
+
+### How to bypass precommit hooks?
+
+To exclude the bandit pre-commit hook when making a Git commit, you can use the --no-verify option. This bypasses any pre-commit hooks that are set up in your repository.
+
+```sh
+git commit --no-verify -m "Your commit message"
+```

precommit-setup.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Function to add the installation path to PATH
+add_to_path() {
+    if [[ ":$PATH:" != *":$1:"* ]]; then
+        export PATH="$PATH:$1"
+        echo "Added $1 to PATH"
+    else
+        echo "$1 is already in PATH"
+    fi
+}
+
+# Function to check if Python and pip are installed
+check_python_and_pip() {
+    if ! command -v python3 &> /dev/null; then
+        echo "Python3 is not installed. Please install Python3 and try again."
+        exit 1
+    fi
+
+    if ! command -v pip &> /dev/null; then
+        echo "pip is not installed. Please install pip and try again."
+        exit 1
+    fi
+}
+
+# Function to install pre-commit
+install_precommit() {
+    if ! command -v pre-commit &> /dev/null; then
+        echo "pre-commit not found, installing..."
+        pip install --user pre-commit
+    else
+        echo "pre-commit is already installed"
+    fi
+}
+
+# Check if Python and pip are installed
+check_python_and_pip
+
+# Detect the operating system
+OS="$(uname -s)"
+case "$OS" in
+    Linux*)     
+        echo "Detected Linux"
+        INSTALL_PATH="$HOME/.local/bin"
+        install_precommit
+        add_to_path "$INSTALL_PATH"
+        ;;
+    Darwin*)    
+        echo "Detected MacOS"
+        INSTALL_PATH="$HOME/.local/bin"
+        install_precommit
+        add_to_path "$INSTALL_PATH"
+        ;;
+    CYGWIN*|MINGW32*|MSYS*|MINGW*)
+        echo "Detected Windows"
+        INSTALL_PATH="$HOME/AppData/Local/Packages/PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0/LocalCache/local-packages/Python312/Scripts"
+        install_precommit
+        add_to_path "$INSTALL_PATH"
+        ;;
+    *)
+        echo "Unknown OS"
+        exit 1
+        ;;
+esac
+
+# Add the installation path to the shell profile for persistence
+if [[ "$OS" == "Linux" || "$OS" == "Darwin" ]]; then
+    SHELL_PROFILE="$HOME/.bashrc"
+    if [[ -f "$HOME/.zshrc" ]]; then
+        SHELL_PROFILE="$HOME/.zshrc"
+    fi
+    echo "export PATH=\$PATH:$INSTALL_PATH" >> "$SHELL_PROFILE"
+    source "$SHELL_PROFILE"
+elif [[ "$OS" == "CYGWIN"* || "$OS" == "MINGW"* || "$OS" == "MSYS"* ]]; then
+    SHELL_PROFILE="$HOME/.bash_profile"
+    echo "export PATH=\$PATH:$INSTALL_PATH" >> "$SHELL_PROFILE"
+    source "$SHELL_PROFILE"
+fi
+
+# Verify the installation
+if command -v pre-commit &> /dev/null; then
+    echo "pre-commit installation successful"
+    pre-commit --version
+else
+    echo "pre-commit installation failed"
+    exit 1
+fi

scripts/lint.sh

@@ -4,7 +4,7 @@ set -Eeuo pipefail
 base_dir=$(dirname $(dirname $0))
 
 # Run the pre-commit checks
-pre-commit run --all-files
+SKIP=bandit pre-commit run --all-files
 
 ruff check --config "${base_dir}/pyproject.toml" openfl/
 

tests/openfl/federated/plan/plan_example.yaml

@@ -9,6 +9,7 @@ aggregator :
     best_state_path : save/best.pbuf
     last_state_path : save/last.pbuf
     rounds_to_train : 10
+    persistent_db_path: tensor.db
 
 collaborator :
   defaults : plan/defaults/collaborator.yaml