From d29484ad1161c30cd3ba7d8490fe609282e05158 Mon Sep 17 00:00:00 2001 From: Teodor Parvanov Date: Mon, 21 Oct 2024 15:44:54 +0200 Subject: [PATCH 01/19] Fixing manual PKI setup commands in taskrunner.rst Signed-off-by: Teodor Parvanov --- docs/about/features_index/taskrunner.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index 034163de02..4e67622088 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -303,11 +303,11 @@ Setting Up the Certificate Authority .. note:: - You can override the apparent FQDN of the system by setting an FQDN environment variable before creating the certificate. + You can override the apparent FQDN by setting it explicitly via the :code:`--fqdn` parameter. .. code-block:: console - $ fx aggregator generate-cert-request export FQDN=x.x.x.x + $ fx aggregator generate-cert-request --fqdn AFQDN If you omit the :code:`--fdqn` parameter, then :code:`fx` will automatically use the FQDN of the current node assuming the node has been correctly set with a static address. @@ -324,11 +324,11 @@ Setting Up the Certificate Authority .. note:: - You can override the apparent FQDN of the system by setting an FQDN environment variable (:code:`export FQDN=x.x.x.x`) before signing the certificate. + You can override the apparent FQDN of the system by setting an FQDN environment variable (:code:`export FQDN=AFQDN`) before signing the certificate. .. code-block:: console - $ fx aggregator certify export FQDN=x.x.x.x + $ fx aggregator certify --fqdn AFQDN 5. This node now has a signed security certificate as the aggregator for this new federation. You should have the following files. From 7933bdc810144ab92bfb942637d9ea4eb47e172d Mon Sep 17 00:00:00 2001 From: noopur Date: Tue, 22 Oct 2024 09:23:54 +0000 Subject: [PATCH 02/19] Logging fix - No of rounds saved vs started Signed-off-by: noopur --- .github/workflows/pytest_coverage.yml | 3 ++- openfl/component/aggregator/aggregator.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml index 32e89ceb69..6d50288ca8 100644 --- a/.github/workflows/pytest_coverage.yml +++ b/.github/workflows/pytest_coverage.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python +# This workflow will run code coverage # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Pytest and code coverage @@ -6,6 +6,7 @@ name: Pytest and code coverage on: pull_request: branches: [ develop ] + workflow_dispatch: permissions: contents: read diff --git a/openfl/component/aggregator/aggregator.py b/openfl/component/aggregator/aggregator.py index 81d3e7411a..0ec816276b 100644 --- a/openfl/component/aggregator/aggregator.py +++ b/openfl/component/aggregator/aggregator.py @@ -968,16 +968,17 @@ def _end_of_round_check(self): # Once all of the task results have been processed self._end_of_round_check_done[self.round_number] = True + + # Save the latest model + self.logger.info("Saving round %s model...", self.round_number) + self._save_model(self.round_number, self.last_state_path) + self.round_number += 1 # resetting stragglers for task for a new round self.stragglers = [] # resetting collaborators_done for next round self.collaborators_done = [] - # Save the latest model - self.logger.info("Saving round %s model...", self.round_number) - self._save_model(self.round_number, self.last_state_path) - # TODO This needs to be fixed! if self._time_to_quit(): self.logger.info("Experiment Completed. Cleaning up...") From bc128407d14ef1058ff6d143a28fff938be1ea7a Mon Sep 17 00:00:00 2001 From: rajith Date: Thu, 24 Oct 2024 11:27:59 +0530 Subject: [PATCH 03/19] fix filename coverity issue --- openfl/interface/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openfl/interface/cli.py b/openfl/interface/cli.py index 9d2b1c1b8d..6b22351b97 100755 --- a/openfl/interface/cli.py +++ b/openfl/interface/cli.py @@ -4,6 +4,7 @@ """CLI module.""" import logging import os +import re import sys import time import warnings @@ -181,6 +182,9 @@ def cli(context, log_level, no_warnings): # This will be overridden later with user selected debugging level disable_warnings() log_file = os.getenv("LOG_FILE") + # Validate log_file using allow list approach + if log_file and not re.match(r'^[\w\-.]+$', log_file): + raise ValueError("Invalid log file path") setup_logging(log_level, log_file) sys.stdout.reconfigure(encoding="utf-8") From b529f8f2c8affe639a860fabe1aeae2d1f1e18a6 Mon Sep 17 00:00:00 2001 From: rajith Date: Thu, 24 Oct 2024 11:46:22 +0530 Subject: [PATCH 04/19] fix code format issue --- openfl/interface/cli.py | 2 +- openfl/utilities/optimizers/numpy/yogi_optimizer.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openfl/interface/cli.py b/openfl/interface/cli.py index 6b22351b97..ba7b9c2133 100755 --- a/openfl/interface/cli.py +++ b/openfl/interface/cli.py @@ -183,7 +183,7 @@ def cli(context, log_level, no_warnings): disable_warnings() log_file = os.getenv("LOG_FILE") # Validate log_file using allow list approach - if log_file and not re.match(r'^[\w\-.]+$', log_file): + if log_file and not re.match(r"^[\w\-.]+$", log_file): raise ValueError("Invalid log file path") setup_logging(log_level, log_file) sys.stdout.reconfigure(encoding="utf-8") diff --git a/openfl/utilities/optimizers/numpy/yogi_optimizer.py b/openfl/utilities/optimizers/numpy/yogi_optimizer.py index 17f9fc85a2..5bce181d50 100644 --- a/openfl/utilities/optimizers/numpy/yogi_optimizer.py +++ b/openfl/utilities/optimizers/numpy/yogi_optimizer.py @@ -79,7 +79,8 @@ def _update_second_moment(self, grad_name: str, grad: np.ndarray) -> None: """ sign = np.sign(grad**2 - self.grads_second_moment[grad_name]) self.grads_second_moment[grad_name] = ( - self.beta_2 * self.grads_second_moment[grad_name] + (1.0 - self.beta_2) * sign * grad**2 + self.beta_2 * self.grads_second_moment[grad_name] + + (1.0 - self.beta_2) * sign * grad**2 ) def step(self, gradients: Dict[str, np.ndarray]) -> None: From 6c9c0029603fe2146332aea45145ba404d3c733a Mon Sep 17 00:00:00 2001 From: rajith Date: Thu, 24 Oct 2024 12:03:37 +0530 Subject: [PATCH 05/19] reverting code check format issue --- openfl/utilities/optimizers/numpy/yogi_optimizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openfl/utilities/optimizers/numpy/yogi_optimizer.py b/openfl/utilities/optimizers/numpy/yogi_optimizer.py index 5bce181d50..17f9fc85a2 100644 --- a/openfl/utilities/optimizers/numpy/yogi_optimizer.py +++ b/openfl/utilities/optimizers/numpy/yogi_optimizer.py @@ -79,8 +79,7 @@ def _update_second_moment(self, grad_name: str, grad: np.ndarray) -> None: """ sign = np.sign(grad**2 - self.grads_second_moment[grad_name]) self.grads_second_moment[grad_name] = ( - self.beta_2 * self.grads_second_moment[grad_name] - + (1.0 - self.beta_2) * sign * grad**2 + self.beta_2 * self.grads_second_moment[grad_name] + (1.0 - self.beta_2) * sign * grad**2 ) def step(self, gradients: Dict[str, np.ndarray]) -> None: From 28e58ae42263ffc6a8a649cbfb8026e951d5d4ea Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 08:50:45 +0000 Subject: [PATCH 06/19] Adding Workflow_Interface_101_MNIST To CI Pipeline Signed-off-by: noopur --- .../workflow_interface_101_mnist.yml | 39 +++++++++++++++++++ .../Workflow_Interface_101_MNIST.ipynb | 4 ++ 2 files changed, 43 insertions(+) create mode 100644 .github/workflows/.github/workflows/workflow_interface_101_mnist.yml diff --git a/.github/workflows/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/.github/workflows/workflow_interface_101_mnist.yml new file mode 100644 index 0000000000..d8764ffd10 --- /dev/null +++ b/.github/workflows/.github/workflows/workflow_interface_101_mnist.yml @@ -0,0 +1,39 @@ +#--------------------------------------------------------------------------- +# Workflow to run 101 MNIST Notebook +# Authors - Noopur, Payal Chaurasiya +#--------------------------------------------------------------------------- +name: Workflow Interface 101 MNIST Notebook + +on: + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout OpenFL repository + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 2 # needed for detecting changes + submodules: "true" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install Jupyter Lab Package + run: | + pip install jupyterlab + - name: Run Notebook + run: | + jupyter nbconvert --execute --to notebook ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb + echo "Notebook run completed" + tar -cvf notebook.tar ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.nbconvert.ipynb + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} # collect artifacts regardless of failures + with: + name: wf_interface_101_mnist_${{ github.run_id }} + path: notebook.tar diff --git a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb index edcb601793..9417923188 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb @@ -67,6 +67,10 @@ "metadata": {}, "outputs": [], "source": [ + "# Below code will display the print statement output on screen as well\n", + "import sys\n", + "sys.stdout = open('/dev/stdout', 'w')\n", + "\n", "!pip install git+https://github.com/securefederatedai/openfl.git\n", "!pip install -r workflow_interface_requirements.txt\n", "!pip install torch\n", From 05989ec9dde42803608d9935724b23921bd9f8f0 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 08:53:25 +0000 Subject: [PATCH 07/19] Corrected the path Signed-off-by: noopur --- .../{.github/workflows => }/workflow_interface_101_mnist.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{.github/workflows => }/workflow_interface_101_mnist.yml (100%) diff --git a/.github/workflows/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml similarity index 100% rename from .github/workflows/.github/workflows/workflow_interface_101_mnist.yml rename to .github/workflows/workflow_interface_101_mnist.yml From 9757bfca159dfde4551187e35e0e3360ea7a8660 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 09:24:34 +0000 Subject: [PATCH 08/19] Run for multiple versions Signed-off-by: noopur --- .github/workflows/workflow_interface_101_mnist.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index d8764ffd10..e9d8d94ac7 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -9,7 +9,12 @@ on: jobs: run: + # Run this workflow for various Python versions + strategy: + matrix: + python-version: ['3.8','3.9','3.10','3.11'] runs-on: ubuntu-latest + steps: - name: Checkout OpenFL repository uses: actions/checkout@v4.1.1 @@ -21,16 +26,20 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: ${{ matrix.python-version }} - name: Install Jupyter Lab Package run: | pip install jupyterlab + - name: Run Notebook run: | jupyter nbconvert --execute --to notebook ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb echo "Notebook run completed" - tar -cvf notebook.tar ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.nbconvert.ipynb + + - name: Tar files + run: tar -cvf notebook.tar ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.nbconvert.ipynb + - name: Upload Artifacts uses: actions/upload-artifact@v4 if: ${{ always() }} # collect artifacts regardless of failures From 9c3e6794f4bf9a7cd8409606083e80d1597d1a52 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 09:35:42 +0000 Subject: [PATCH 09/19] Ubuntu 22.04 and python 3.10 Signed-off-by: noopur --- .github/workflows/workflow_interface_101_mnist.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index e9d8d94ac7..56b1ccfda5 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -8,13 +8,8 @@ on: workflow_dispatch: jobs: - run: - # Run this workflow for various Python versions - strategy: - matrix: - python-version: ['3.8','3.9','3.10','3.11'] - runs-on: ubuntu-latest - + run_notebook: + runs-on: ubuntu-22.04 steps: - name: Checkout OpenFL repository uses: actions/checkout@v4.1.1 @@ -26,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: ${{ matrix.python-version }} + python-version: 3.10 - name: Install Jupyter Lab Package run: | From d31978242cd3613b836ec342812b0c8dc9282e72 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 09:40:33 +0000 Subject: [PATCH 10/19] 3.10 as string Signed-off-by: noopur --- .github/workflows/workflow_interface_101_mnist.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index 56b1ccfda5..ddd5619e24 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -21,11 +21,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: Install Jupyter Lab Package - run: | - pip install jupyterlab + run: pip install jupyterlab - name: Run Notebook run: | From acc0fa6921094fbdd236dd8c4a4d8e49b4758d19 Mon Sep 17 00:00:00 2001 From: noopur Date: Thu, 24 Oct 2024 11:58:10 +0000 Subject: [PATCH 11/19] Enable Workflow Interface 101 MNIST For PR Pipeline Signed-off-by: noopur --- .github/workflows/workflow_interface_101_mnist.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index ddd5619e24..aa1c198fb8 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -5,8 +5,14 @@ name: Workflow Interface 101 MNIST Notebook on: + pull_request: + branches: [ develop ] + workflow_dispatch: +permissions: + contents: read + jobs: run_notebook: runs-on: ubuntu-22.04 From 9d13d9bf76319dd098aaed249be6f85502f08d4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:45:04 +0000 Subject: [PATCH 12/19] build(deps): bump onnx in /openfl-workspace/gandlf_seg_test Bumps [onnx](https://github.com/onnx/onnx) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/onnx/onnx/releases) - [Changelog](https://github.com/onnx/onnx/blob/main/docs/Changelog-ml.md) - [Commits](https://github.com/onnx/onnx/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: onnx dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- openfl-workspace/gandlf_seg_test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfl-workspace/gandlf_seg_test/requirements.txt b/openfl-workspace/gandlf_seg_test/requirements.txt index fb498426ce..baae23bc80 100644 --- a/openfl-workspace/gandlf_seg_test/requirements.txt +++ b/openfl-workspace/gandlf_seg_test/requirements.txt @@ -1 +1 @@ -onnx==1.16.0 +onnx==1.17.0 From 46a3f72f73596a223de6e1ebe2afb54fc6b6cc4a Mon Sep 17 00:00:00 2001 From: rajith Date: Fri, 25 Oct 2024 11:11:46 +0530 Subject: [PATCH 13/19] fix coverity hash issue in metaflow_utils file --- openfl/experimental/utilities/metaflow_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index 066429a41e..e13d4f1f90 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -66,9 +66,9 @@ def __init__(self, name): def __enter__(self): lock_id = hashlib.new( - "md5", self.name.encode("utf8"), usedforsecurity=False + "sha256", self.name.encode("utf8"), usedforsecurity=False ).hexdigest() # nosec - # MD5sum used for concurrency purposes, not security + # Using SHA-256 to address security warning self.fp = open(f"/tmp/.lock-{lock_id}.lck", "wb") fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) @@ -345,11 +345,7 @@ def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): def pickle_iter(): for name, obj in artifacts_iter: - do_v4 = ( - force_v4 and force_v4 - if isinstance(force_v4, bool) - else force_v4.get(name, False) - ) + do_v4 = force_v4 if isinstance(force_v4, bool) else force_v4.get(name, False) if do_v4: encode_type = "gzip+pickle-v4" if encode_type not in self._encodings: From ccca88e66af3589936b2dbc21c1e6be861ae009d Mon Sep 17 00:00:00 2001 From: Shailesh Tanwar <135304487+tanwarsh@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:44:54 +0530 Subject: [PATCH 14/19] Added ipywidgets (#1103) * added ipywidgets and solution for MetaflowUnknownUser Signed-off-by: yes * reverting sppace change Signed-off-by: yes * added ipywidgets and solution for MetaflowUnknownUser Signed-off-by: yes * reverting sppace change Signed-off-by: yes * comment changes Signed-off-by: yes * comment changes Signed-off-by: yes --------- Signed-off-by: yes --- .../experimental/Workflow_Interface_101_MNIST.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb index 9417923188..f429bbb157 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb @@ -75,8 +75,9 @@ "!pip install -r workflow_interface_requirements.txt\n", "!pip install torch\n", "!pip install torchvision\n", + "!pip install -U ipywidgets\n", "\n", - "# Uncomment this if running in Google Colab\n", + "# Uncomment this if running in Google Colab and set USERNAME if running in docker container.\n", "# !pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/workflow_interface_requirements.txt\n", "# import os\n", "# os.environ[\"USERNAME\"] = \"colab\"" @@ -390,9 +391,8 @@ ] }, { - "attachments": {}, "cell_type": "markdown", - "id": "86b3dd2e", + "id": "9a7cc8f7", "metadata": {}, "source": [ "Now that the flow has completed, let's get the final model and accuracy" @@ -689,7 +689,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.19" + "version": "3.11.5" } }, "nbformat": 4, From c41647a96b39699f56de8ac102eb38999014f335 Mon Sep 17 00:00:00 2001 From: rajith Date: Fri, 25 Oct 2024 11:11:46 +0530 Subject: [PATCH 15/19] fix coverity hash issue in metaflow_utils file --- openfl/experimental/utilities/metaflow_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index 066429a41e..e13d4f1f90 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -66,9 +66,9 @@ def __init__(self, name): def __enter__(self): lock_id = hashlib.new( - "md5", self.name.encode("utf8"), usedforsecurity=False + "sha256", self.name.encode("utf8"), usedforsecurity=False ).hexdigest() # nosec - # MD5sum used for concurrency purposes, not security + # Using SHA-256 to address security warning self.fp = open(f"/tmp/.lock-{lock_id}.lck", "wb") fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) @@ -345,11 +345,7 @@ def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): def pickle_iter(): for name, obj in artifacts_iter: - do_v4 = ( - force_v4 and force_v4 - if isinstance(force_v4, bool) - else force_v4.get(name, False) - ) + do_v4 = force_v4 if isinstance(force_v4, bool) else force_v4.get(name, False) if do_v4: encode_type = "gzip+pickle-v4" if encode_type not in self._encodings: From 54c5597aeb34049cc3629d69f5e16b64f30d3185 Mon Sep 17 00:00:00 2001 From: rajith Date: Fri, 25 Oct 2024 13:08:25 +0530 Subject: [PATCH 16/19] dummy commit workflow_interface --- openfl/experimental/utilities/metaflow_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index e13d4f1f90..61f9099fa8 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -345,6 +345,7 @@ def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): def pickle_iter(): for name, obj in artifacts_iter: + #removed extra variable do_v4 = force_v4 if isinstance(force_v4, bool) else force_v4.get(name, False) if do_v4: encode_type = "gzip+pickle-v4" From 743778f72df65769a71e84f4b5d589b5cb3e7d35 Mon Sep 17 00:00:00 2001 From: rajith Date: Fri, 25 Oct 2024 13:19:42 +0530 Subject: [PATCH 17/19] trigger withworkflow_interface --- openfl/experimental/utilities/metaflow_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index 61f9099fa8..e13d4f1f90 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -345,7 +345,6 @@ def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): def pickle_iter(): for name, obj in artifacts_iter: - #removed extra variable do_v4 = force_v4 if isinstance(force_v4, bool) else force_v4.get(name, False) if do_v4: encode_type = "gzip+pickle-v4" From 35c16b11cd57668e4d0594bbe3ccc587cc375848 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Fri, 25 Oct 2024 13:15:08 +0530 Subject: [PATCH 18/19] Rename experimental interface notebooks Signed-off-by: Shah, Karan --- ...k.ipynb => 1001_Workspace_Creation_from_JupyterNotebook.ipynb} | 0 .../{Workflow_Interface_101_MNIST.ipynb => 101_MNIST.ipynb} | 0 ...ggregator_Validation.ipynb => 102_Aggregator_Validation.ipynb} | 0 ....ipynb => 103_Cyclic_Institutional_Incremental_Learning.ipynb} | 0 ..._Keras_MNIST_with_GPU.ipynb => 104_Keras_MNIST_with_GPU.ipynb} | 0 ...Workflow_Interface_104_MNIST_XPU.ipynb => 104_MNIST_XPU.ipynb} | 0 ...sive_GPUs_with_Ray.ipynb => 201_Exclusive_GPUs_with_Ray.ipynb} | 0 ..._301_MNIST_Watermarking.ipynb => 301_MNIST_Watermarking.ipynb} | 0 ...hetic_nonIID.ipynb => 401_FedProx_with_Synthetic_nonIID.ipynb} | 0 ...ynb => 401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb} | 0 .../{Workflow_Interface_Vertical_FL.ipynb => Intro.ipynb} | 0 .../{Workflow_Interface_VFL_Two_Party.ipynb => TwoParty.ipynb} | 0 ...from_JupyterNotebook.ipynb => TwoPartyWorkspaceCreation.ipynb} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename openfl-tutorials/experimental/{Workflow_Interface_1001_Workspace_Creation_from_JupyterNotebook.ipynb => 1001_Workspace_Creation_from_JupyterNotebook.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_101_MNIST.ipynb => 101_MNIST.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_102_Aggregator_Validation.ipynb => 102_Aggregator_Validation.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb => 103_Cyclic_Institutional_Incremental_Learning.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb => 104_Keras_MNIST_with_GPU.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_104_MNIST_XPU.ipynb => 104_MNIST_XPU.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb => 201_Exclusive_GPUs_with_Ray.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_301_MNIST_Watermarking.ipynb => 301_MNIST_Watermarking.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb => 401_FedProx_with_Synthetic_nonIID.ipynb} (100%) rename openfl-tutorials/experimental/{Workflow_Interface_401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb => 401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb} (100%) rename openfl-tutorials/experimental/Vertical_FL/{Workflow_Interface_Vertical_FL.ipynb => Intro.ipynb} (100%) rename openfl-tutorials/experimental/Vertical_FL/{Workflow_Interface_VFL_Two_Party.ipynb => TwoParty.ipynb} (100%) rename openfl-tutorials/experimental/Vertical_FL/{Workflow_Interface_VFL_Two_Party_Workspace_Creation_from_JupyterNotebook.ipynb => TwoPartyWorkspaceCreation.ipynb} (100%) diff --git a/openfl-tutorials/experimental/Workflow_Interface_1001_Workspace_Creation_from_JupyterNotebook.ipynb b/openfl-tutorials/experimental/1001_Workspace_Creation_from_JupyterNotebook.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_1001_Workspace_Creation_from_JupyterNotebook.ipynb rename to openfl-tutorials/experimental/1001_Workspace_Creation_from_JupyterNotebook.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb b/openfl-tutorials/experimental/101_MNIST.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb rename to openfl-tutorials/experimental/101_MNIST.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb b/openfl-tutorials/experimental/102_Aggregator_Validation.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb rename to openfl-tutorials/experimental/102_Aggregator_Validation.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb b/openfl-tutorials/experimental/103_Cyclic_Institutional_Incremental_Learning.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb rename to openfl-tutorials/experimental/103_Cyclic_Institutional_Incremental_Learning.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb b/openfl-tutorials/experimental/104_Keras_MNIST_with_GPU.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb rename to openfl-tutorials/experimental/104_Keras_MNIST_with_GPU.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_104_MNIST_XPU.ipynb b/openfl-tutorials/experimental/104_MNIST_XPU.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_104_MNIST_XPU.ipynb rename to openfl-tutorials/experimental/104_MNIST_XPU.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb b/openfl-tutorials/experimental/201_Exclusive_GPUs_with_Ray.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb rename to openfl-tutorials/experimental/201_Exclusive_GPUs_with_Ray.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb b/openfl-tutorials/experimental/301_MNIST_Watermarking.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb rename to openfl-tutorials/experimental/301_MNIST_Watermarking.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb b/openfl-tutorials/experimental/401_FedProx_with_Synthetic_nonIID.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb rename to openfl-tutorials/experimental/401_FedProx_with_Synthetic_nonIID.ipynb diff --git a/openfl-tutorials/experimental/Workflow_Interface_401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb b/openfl-tutorials/experimental/401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb similarity index 100% rename from openfl-tutorials/experimental/Workflow_Interface_401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb rename to openfl-tutorials/experimental/401_MNIST_Aggregator_Validation_Ray_Watermarking.ipynb diff --git a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb b/openfl-tutorials/experimental/Vertical_FL/Intro.ipynb similarity index 100% rename from openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb rename to openfl-tutorials/experimental/Vertical_FL/Intro.ipynb diff --git a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb b/openfl-tutorials/experimental/Vertical_FL/TwoParty.ipynb similarity index 100% rename from openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb rename to openfl-tutorials/experimental/Vertical_FL/TwoParty.ipynb diff --git a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party_Workspace_Creation_from_JupyterNotebook.ipynb b/openfl-tutorials/experimental/Vertical_FL/TwoPartyWorkspaceCreation.ipynb similarity index 100% rename from openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party_Workspace_Creation_from_JupyterNotebook.ipynb rename to openfl-tutorials/experimental/Vertical_FL/TwoPartyWorkspaceCreation.ipynb From 1b051b05c6ebba310999b517d8b8f88272f140af Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Fri, 25 Oct 2024 13:20:28 +0530 Subject: [PATCH 19/19] Update CI test Signed-off-by: Shah, Karan --- .github/workflows/workflow_interface_101_mnist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml index aa1c198fb8..1f2c75c95a 100644 --- a/.github/workflows/workflow_interface_101_mnist.yml +++ b/.github/workflows/workflow_interface_101_mnist.yml @@ -34,11 +34,11 @@ jobs: - name: Run Notebook run: | - jupyter nbconvert --execute --to notebook ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb + jupyter nbconvert --execute --to notebook ./openfl-tutorials/experimental/101_MNIST.ipynb echo "Notebook run completed" - name: Tar files - run: tar -cvf notebook.tar ./openfl-tutorials/experimental/Workflow_Interface_101_MNIST.nbconvert.ipynb + run: tar -cvf notebook.tar ./openfl-tutorials/experimental/101_MNIST.nbconvert.ipynb - name: Upload Artifacts uses: actions/upload-artifact@v4