Merge branch 'develop' into llm_phi

.github/workflows/docker-bench-security.yml

@@ -2,7 +2,6 @@ name: Docker Bench for Security
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/double_ws_export.yml

@@ -5,7 +5,6 @@ name: Double workspace export
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/experimental_workflow_tests.yml

@@ -2,9 +2,10 @@ name: Workflow Interface Tests
 
 on:
   push:
-    branches: [ develop ]
+    branches: 
+      - develop
+      - v1.7.x
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/gandlf.yml

@@ -5,7 +5,6 @@ name: GaNDLF TaskRunner
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/hadolint.yml

@@ -5,7 +5,6 @@ name: Hadolint Security Scan
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/interactive-kvasir.yml

@@ -5,7 +5,6 @@ name: Interactive API - Pytorch Kvasir UNet
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/interactive-tensorflow.yml

@@ -5,7 +5,6 @@ name: Interactive API - Tensorflow MNIST
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/lint.yml

@@ -5,7 +5,6 @@ name: Check code format
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/openfl-docker-build.yml

@@ -0,0 +1,47 @@
+name: Build and Push Openfl Docker Image
+
+on:
+  push:
+    branches: 
+      - develop
+      - v1.7.x
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to tag the Docker image with'
+        required: false
+        default: 'latest'
+
+env:
+  VERSION: ${{ github.ref == 'refs/heads/develop' && 'latest' || '1.7' }}
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:      
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build and Push OpenFL Base Image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: openfl-docker/Dockerfile.base
+        push: true
+        tags: |
+          ghcr.io/${{ github.repository }}/openfl:${{ github.event.inputs.version || 'latest' }}

.github/workflows/pki.yml

@@ -5,7 +5,6 @@ name: Private Key Infrastructure
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/pytest_coverage.yml

@@ -5,7 +5,6 @@ name: Pytest and code coverage
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
 

.github/workflows/straggler-handling.yml

@@ -5,7 +5,6 @@ name: Straggler Handling Test
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/taskrunner.yml

@@ -5,7 +5,6 @@ name: TaskRunner
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/taskrunner_eden_pipeline.yml

@@ -5,7 +5,6 @@ name: TaskRunner (Eden Compression)
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/tr_docker_gramine_direct.yml

@@ -3,7 +3,6 @@ name: TaskRunner (docker/gramine-direct)
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/tr_docker_native.yml

@@ -3,7 +3,6 @@ name: TaskRunner (docker/native)
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
 permissions:

.github/workflows/trivy.yml

@@ -1,9 +1,10 @@
 name: Trivy
 on:
   push:
-    branches: [ develop ]
+    branches: 
+      - develop
+      - v1.7.x
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 jobs:
   build:

.github/workflows/wf_functional_e2e.yml

@@ -5,7 +5,6 @@ name: Workflow_Functional_E2E
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
   workflow_dispatch:

.github/workflows/workflow_interface_101_mnist.yml

@@ -6,7 +6,6 @@ name: Workflow Interface 101 MNIST Notebook
 
 on:
   pull_request:
-    branches: [ develop ]
     types: [opened, synchronize, reopened, ready_for_review]
 
   workflow_dispatch:

test-requirements.txt

@@ -2,5 +2,8 @@ docker
 lxml==5.3.0
 paramiko
 pytest==8.3.4
-pytest-asyncio==0.25.0
+pytest-asyncio==0.25.1
 pytest-mock==3.14.0
+defusedxml==0.7.1
+matplotlib==3.10.0
+fpdf==1.7.2

tests/end_to_end/models/model_owner.py

@@ -145,6 +145,9 @@ def modify_plan(self, param_config, plan_path):
             # Memory Leak related
             data["aggregator"]["settings"]["log_memory_usage"] = self.log_memory_usage
             data["collaborator"]["settings"]["log_memory_usage"] = self.log_memory_usage
+            # Aggregator and collaborators metric logging related
+            data["aggregator"]["settings"]["write_logs"] = True
+            data["collaborator"]["settings"]["write_logs"] = True
 
             data["data_loader"]["settings"]["collaborator_count"] = int(self.num_collaborators)
             data["network"]["settings"]["require_client_auth"] = param_config.require_client_auth

tests/end_to_end/test_suites/memory_logs_tests.py

@@ -4,11 +4,11 @@
 import pytest
 import logging
 import os
-import json
 
 from tests.end_to_end.utils.common_fixtures import fx_federation_tr, fx_federation_tr_dws
 import tests.end_to_end.utils.constants as constants
 from tests.end_to_end.utils import federation_helper as fed_helper, ssh_helper as ssh
+from tests.end_to_end.utils.generate_report import generate_memory_report, convert_to_json
 
 log = logging.getLogger(__name__)
 
@@ -78,7 +78,9 @@ def _log_memory_usage(request, fed_obj):
     ), "Aggregator memory usage file is not available"
 
     # Log the aggregator memory usage details
-    memory_usage_dict = json.load(open(aggregator_memory_usage_file))
+    memory_usage_dict = convert_to_json(aggregator_memory_usage_file)
+    aggregator_path = os.path.join(fed_obj.workspace_path, "aggregator")
+    generate_memory_report(memory_usage_dict, aggregator_path)
 
     # check memory usage entries for each round
     assert (
@@ -98,7 +100,9 @@ def _log_memory_usage(request, fed_obj):
             collaborator_memory_usage_file
         ), f"Memory usage file for collaborator {collaborator.collaborator_name} is not available"
 
-        memory_usage_dict = json.load(open(collaborator_memory_usage_file))
+        memory_usage_dict = convert_to_json(collaborator_memory_usage_file)
+        collaborator_path = os.path.join(fed_obj.workspace_path, collaborator.name)
+        generate_memory_report(memory_usage_dict, collaborator_path)
 
         assert (
             len(memory_usage_dict) == request.config.num_rounds

tests/end_to_end/utils/generate_report.py

@@ -0,0 +1,127 @@
+# Copyright 2020-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import json
+from scipy.stats import linregress
+from fpdf import FPDF
+
+
+class PDF(FPDF):
+    def header(self):
+        self.set_font("Arial", "B", 14)
+
+    def chapter_title(self, title):
+        self.add_page()
+        self.set_font("Arial", "B", 14)  # Set font to bold for title
+        self.cell(0, 10, title, 0, 1, "L")
+
+    def chapter_body(self, body):
+        self.set_font("Arial", "", 12)
+        self.multi_cell(0, 10, body)
+
+
+def generate_memory_report(memory_usage_dict, workspace_path):
+    """
+    Generates a memory usage report from a CSV file.
+
+    Parameters:
+    file_path (str): The path to the CSV file containing memory usage data.
+
+    Returns:
+    None
+    """
+    # Load data
+    data = pd.DataFrame(memory_usage_dict)
+
+    # Plotting the chart
+    plt.figure(figsize=(10, 5))
+    plt.plot(data["round_number"], data["virtual_memory/used"], marker="o")
+    plt.title("Memory Usage per Round")
+    plt.xlabel("round_number")
+    plt.ylabel("Virtual Memory Used (MB)")
+    plt.grid(True)
+    output_path = f"{workspace_path}/mem_usage_plot.png"
+    plt.savefig(output_path)
+    plt.close()
+
+    # Calculate statistics
+    min_mem = round(data["virtual_memory/used"].min(), 2)
+    max_mem = round(data["virtual_memory/used"].max(), 2)
+    mean_mem = round(data["virtual_memory/used"].mean(), 2)
+    variance_mem = round(data["virtual_memory/used"].var(), 2)
+    std_dev_mem = round(data["virtual_memory/used"].std(), 2)
+    slope, _, _, _, _ = linregress(data.index, data["virtual_memory/used"])
+    slope = round(slope, 2)
+    stats_path = f"{workspace_path}/mem_stats.txt"
+    with open(stats_path, "w") as file:
+        file.write(f"Minimum Memory Used: {min_mem} MB\n")
+        file.write(f"Maximum Memory Used: {max_mem} MB\n")
+        file.write(f"Mean Memory Used: {mean_mem} MB\n")
+        file.write(f"Variance: {variance_mem}\n")
+        file.write(f"Standard Deviation: {std_dev_mem}\n")
+        file.write(f"Slope: {slope}\n")
+
+    # Generate PDF report
+    pdf = PDF()
+    add_introduction(pdf)
+    add_chart_analysis(pdf, output_path, data)
+    add_statistical_overview(pdf, stats_path)
+    add_conclusion(pdf, slope)
+    pdf_output_path = f"{workspace_path}/MemAnalysis.pdf"
+    pdf.output(pdf_output_path)
+
+    print("Memory report generation completed. Report saved to:", pdf_output_path)
+
+
+def add_introduction(pdf):
+    pdf.chapter_title("Introduction")
+    intro_text = (
+        "The purpose of this memory analysis is to identify memory usage trends and potential bottlenecks. "
+        "This analysis focuses on the relationship between round information and memory usage."
+    )
+    pdf.chapter_body(intro_text)
+
+
+def add_chart_analysis(pdf, output_path, data):
+    pdf.chapter_title("Chart Analysis")
+    pdf.image(output_path, w=180)
+    diffs = data["virtual_memory/used"].diff().round(2)
+    significant_changes = diffs[diffs.abs() > 500]
+    for index, value in significant_changes.items():
+        pdf.chapter_body(
+            f"Significant memory change: {value} MB at Round {data['round_number'][index]}"
+        )
+
+
+def add_statistical_overview(pdf, stats_path):
+    pdf.chapter_title("Statistical Overview")
+    with open(stats_path, "r") as file:
+        stats = file.read()
+    pdf.chapter_body(stats)
+
+
+def add_conclusion(pdf, slope):
+    pdf.chapter_title("Conclusion")
+    if slope > 0:
+        conclusion_text = "The upward slope in the graph indicates a trend of increasing memory usage over rounds."
+    else:
+        conclusion_text = "There is no continuous memory growth."
+    pdf.chapter_body(conclusion_text)
+
+
+def convert_to_json(file):
+    """
+    Reads a file containing JSON objects, one per line, and converts them into a list of parsed JSON objects.
+    Args:
+        file (str): The path to the file containing JSON objects.
+    Returns:
+        list: A list of parsed JSON objects.
+    """
+    with open(file, "r") as infile:
+        json_objects = infile.readlines()
+
+    # Parse each JSON object
+    parsed_json_objects = [json.loads(obj) for obj in json_objects]
+    return parsed_json_objects