Merge branch 'main' into app-routes-follow-up

# Conflicts:
#	frontend/src/components/drawers/navigation/MainNav.vue

.github/workflows/branch-deploy.yaml

@@ -14,6 +14,14 @@ on:
         description: 'flowfuse/nr-launcher branch name'
         required: true
         default: 'main'
+      nr_project_nodes_branch:
+        description: 'flowfuse/nr-project-nodes branch name'
+        required: true
+        default: 'main'
+      nr_file_nodes_branch:
+        description: 'flowfuse/nr-file-nodes branch name'
+        required: true
+        default: 'main'
   pull_request:
     types: 
       - opened
@@ -67,7 +75,6 @@ jobs:
     if: |
       needs.validate-user.outputs.is_org_member == 'true' &&
       github.event_name == 'workflow_dispatch' &&
-      github.event.action != 'closed' &&
       inputs.driver_k8s_branch != 'main'
     uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
     with:
@@ -79,21 +86,61 @@ jobs:
     secrets:
       npm_registry_token: ${{ secrets.NPM_PUBLISH_TOKEN }}
 
-  publish_nr_launcher:
-    name: Build and publish Node-RED launcher package
+  publish_nr_project_nodes:
+    name: Build and publish nr-project-nodes package
+    needs: validate-user
+    if: |
+      needs.validate-user.outputs.is_org_member == 'true' &&
+      github.event_name == 'workflow_dispatch' &&
+      inputs.nr_project_nodes_branch != 'main'
+    uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
+    with:
+      package_name: nr-project-nodes
+      publish_package: true
+      repository_name: 'FlowFuse/nr-project-nodes'
+      branch_name: ${{ inputs.nr_project_nodes_branch }}
+      release_name: "pre-staging-${{ inputs.nr_project_nodes_branch }}"
+    secrets:
+      npm_registry_token: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
+  publish_nr_file_nodes:
+    name: Build and publish nr-file-nodes package
     needs: validate-user
     if: |
       needs.validate-user.outputs.is_org_member == 'true' &&
       github.event_name == 'workflow_dispatch' &&
-      github.event.action != 'closed' &&
-      inputs.nr_launcher_branch != 'main'
+      inputs.nr_file_nodes_branch != 'main'
+    uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
+    with:
+      package_name: nr-file-nodes
+      publish_package: true
+      repository_name: 'FlowFuse/nr-file-nodes'
+      branch_name: ${{ inputs.nr_file_nodes_branch }}
+      release_name: "pre-staging-${{ inputs.nr_file_nodes_branch }}"
+    secrets:
+      npm_registry_token: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
+  publish_nr_launcher:
+    name: Build and publish nr-launcher package
+    needs: 
+      - validate-user
+      - publish_nr_project_nodes
+      - publish_nr_file_nodes
+    if: |
+      needs.validate-user.outputs.is_org_member == 'true' &&
+      github.event_name == 'workflow_dispatch' &&
+      (always() && inputs.nr_launcher_branch != 'main') || needs.publish_nr_project_nodes.result == 'success' || needs.publish_nr_file_nodes.result == 'success'
     uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
     with:
       package_name: flowfuse-nr-launcher
       publish_package: true
       repository_name: 'FlowFuse/nr-launcher'
       branch_name: ${{ inputs.nr_launcher_branch }}
-      release_name: "pre-staging-${{ inputs.nr_launcher_branch }}"
+      release_name: "pre-staging-${{ inputs.nr_launcher_branch == 'main' && github.sha || inputs.nr_launcher_branch }}"
+      package_dependencies: |
+        @flowfuse/nr-project-nodes=${{ inputs.nr_project_nodes_branch != 'main' && needs.publish_nr_project_nodes.outputs.release_name || 'nightly' }}
+        @flowfuse/nr-file-nodes=${{ inputs.nr_file_nodes_branch != 'main' && needs.publish_nr_file_nodes.outputs.release_name || 'nightly' }}
+        @flowfuse/nr-assistant=nightly
     secrets:
       npm_registry_token: ${{ secrets.NPM_PUBLISH_TOKEN }}
 
@@ -105,15 +152,12 @@ jobs:
     if: |
       needs.validate-user.outputs.is_org_member == 'true' &&
       github.event_name == 'workflow_dispatch' &&
-      github.event.action != 'closed' &&
-      (always() && needs.publish_nr_launcher.result != 'failure')
+      (always() && needs.publish_nr_launcher.result == 'success')
     uses: flowfuse/github-actions-workflows/.github/workflows/[email protected]
     with:
       image_name: 'node-red'
       dockerfile_path: Dockerfile
       image_tag_prefix: '4.0.x-'
-      package_dependencies: |
-        @flowforge/nr-project-nodes=nightly
       build_context: './ci/node-red'
       build_arguments: |
         BUILD_TAG=${{ needs.publish_nr_launcher.outputs.release_name }}
@@ -130,7 +174,7 @@ jobs:
     if: |
       needs.validate-user.outputs.is_org_member == 'true' &&
       github.event_name == 'workflow_dispatch' &&
-      github.event.action != 'closed'
+      (always() && needs.build-node-red.result == 'success')
     runs-on: ubuntu-latest
     environment: staging
     env:

.github/workflows/publish.yml

@@ -25,7 +25,7 @@ jobs:
   check-tests-status:
     name: Check tests statuses
     if: |
-      ( github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/main' ) ||
+      github.ref == 'refs/heads/main' ||
       ( github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' )
     runs-on: ubuntu-latest
     outputs:
@@ -58,18 +58,34 @@ jobs:
             const relevantJobs = data.jobs.filter(job => jobsToCheck.includes(job.name));
             const failedJobs = relevantJobs.filter(job => job.conclusion === 'failure');
             const allSkippedJobs = relevantJobs.every(job => job.conclusion === 'skipped');
+            console.log('Relevant jobs found:', JSON.stringify(relevantJobs, null, 2));
             
             if (failedJobs.length > 0) {
               console.log('🚨 The following tests failed:');
+              await core.summary
+                .addHeading('Failed Tests 🚨');
+              let failedList = '';
               failedJobs.forEach(job => {
                 console.log(`❌ ${job.name}`);
+                failedList += `- ❌ ${job.name}\n`;
               });
+              await core.summary
+                .addRaw(failedList)
+                .write();
               core.setOutput('jobs_status', 'failure');
             } else if (allSkippedJobs) {
               console.log('⏩ All tests were skipped ⏩');
+              await core.summary
+                .addHeading('Test Status ⏩')
+                .addRaw('All tests were skipped')
+                .write();
               core.setOutput('jobs_status', 'skipped');
             } else {
               console.log('✅ Required tests passed successfully ✅');
+              await core.summary
+                .addHeading('Test Status ✅')
+                .addRaw('All required tests passed successfully')
+                .write();
               core.setOutput('jobs_status', 'success');
             }
 

.github/workflows/tests.yml

@@ -209,12 +209,81 @@ jobs:
         actor-map: ${{ vars.SLACK_GITHUB_USERS_MAP }}
         default-mapping: C067BD0377F
 
+    - name: Generate payload variables
+      run: |
+        if [[ "${{ github.ref_name }}" == 'main' || "${{ github.ref_name }}" == 'maintenance' ]] ; then
+          echo "HEADER_MESSAGE=Tests failed against ${{ github.ref_name }} branch" >> $GITHUB_ENV
+          echo "SUMMARY_ICON=no_entry" >> $GITHUB_ENV
+          echo "SUMMARY_MESSAGE= Deployment to FFC environments will not happen until this issue is resolved." >> $GITHUB_ENV
+          echo "LAST_COMMIT_SHA=${{ github.sha}}" >> $GITHUB_ENV
+        else
+          echo "HEADER_MESSAGE=Tests failed against ${{ github.event.number }} pull request" >> $GITHUB_ENV
+          echo "SUMMARY_ICON=warning" >> $GITHUB_ENV
+          echo "SUMMARY_MESSAGE= Please resolve the problem before merging your changes into the main branch." >> $GITHUB_ENV
+          echo "LAST_COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+        fi
+
+
     - name: Send notification
-      uses: ravsamhq/notify-slack-action@v2
+      uses: slackapi/slack-github-action@v2.0.0
       with:
-        status: 'failure'
-        notification_title: 'FlowFuse Tests Pipeline'
-        footer: "<{run_url}|View Run>"
-        mention_users: ${{ steps.map-actor-to-slack.outputs.actor-mapping }}
-    env:
-      SLACK_WEBHOOK_URL: ${{ secrets.GH_WORKFLOWS_WEBHOOK }}
+        method: chat.postMessage
+        token: ${{ secrets.SLACK_GHBOT_TOKEN }}
+        payload: |
+          {
+            "channel": "C067BD0377F",
+            "blocks": [
+              {
+                "type": "header",
+                "text": {
+                  "type": "plain_text",
+                  "text": ":x: ${{ env.HEADER_MESSAGE }}",
+                  "emoji": true
+                }
+              },
+              {
+                "type": "divider"
+              },
+              {
+                "type": "rich_text",
+                "elements": [
+                  {
+                    "type": "rich_text_section",
+                    "elements": [
+                      {
+                        "type": "emoji",
+                        "name": "${{ env.SUMMARY_ICON }}"
+                      },
+                      {
+                        "type": "text",
+                        "text": " ${{ env.SUMMARY_MESSAGE }}",
+                        "style": {
+                          "bold": true
+                        }
+                      }
+                    ]
+                  }
+                ]
+              },
+              {
+                "type": "divider"
+              },
+              {
+                "type": "section",
+                "fields": [
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Author:* <@${{ steps.map-actor-to-slack.outputs.actor-mapping }}>"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "*Last commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ env.LAST_COMMIT_SHA }}|${{ env.LAST_COMMIT_SHA }}>"
+                  },
+                  {
+                    "type": "mrkdwn",
+                    "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
+                  }
+                ]
+              }
+            ]
+          }

docs/user/persistent-context.md

@@ -1,19 +1,53 @@
----
-navTitle: FlowFuse Persistent Context
----
-
 # FlowFuse Persistent Context
 
-Some Node-RED flows need a way to persist context values between restarts and FlowFuse stack 
-updates but context data is ephemeral by default. With FlowFuse Team and Enterprise tiers,
-persistent context storage is possible and it survives restarts, upgrades and more.
+Some Node-RED flows require the ability to persist context values between restarts and FlowFuse stack updates. By default, context data in Node-RED is ephemeral, meaning it does not survive restarts or stack updates. With FlowFuse Starter, Team and Enterprise tiers, however, you can enable **persistent context storage**, ensuring that your context values persist across restarts, upgrades, and more.
 
 ## Usage
 
-In your Node-RED, you will now have the option to store values in 2 context stores:
-* **Memory**: This is for ephemeral context where you do not want it to be persisted.
-* **Persistent**: This is for persistent context where you want values to persist restarts and upgrades.
+In Node-RED with FlowFuse, you now have two context store options:
+
+1. **Memory Context**: This is the default ephemeral context. Values stored in memory are not persistent and will be lost when Node-RED restarts or the Node-RED stack is updated.
+
+2. **Persistent Context**: This allows context values to persist even when Node-RED restarts, updates.
+
+The amount of persistent storage available to you depends on the FlowFuse plan you're subscribed to. FlowFuse offers different storage sizes for each plan, allowing you to select the appropriate level of storage for your needs. For detailed information on storage options, please refer to the [pricing page](https://flowfuse.com/pricing/).
+
+### How to Use FlowFuse Persistent Context
+
+Using persistent context in Node-RED is similar to using memory context, with the key difference being that you specify the storage type for your context data. Here’s how to use persistent context it:
+
+#### Using Persistent Context in Nodes
+
+When configuring persistent context in different nodes (e.g., Change, Inject, or Switch nodes), you can select the type of context storage to use. By default, the context type is set to **Memory**, but you can change it to **Persistent** to store values across restarts.
+
+- **Change Node, Inject Node, Switch Node**:  
+  When you configure these nodes to store or access context data, you’ll notice a storage option at the right corner. By default, it will be set to **Memory**. To make the context **persistent**, simply switch the selection to **Persistent**.
+
+![Persistent Store Option in Change Node](https://flowfuse.com/img/variables-in-node-red-change-node-persistent-store-option--UYd_d_m4p-528.webp)
+
+#### Using Persistent Context in Function Nodes
+
+In **Function nodes**, you interact with context data using the `set` and `get` methods. These methods allow you to specify where the context data should be stored or accessed from.
+
+- **Setting Context**:
+  To set a persistent context value, use the `set` method with three arguments. The third argument specifies the context store, which should be set to **persistent**. For example:
+
+  ```javascript
+  context.set('myKey', 'myValue', 'persistent');
+  ```
+
+This argument is optional and defaults to memory. This means that if you leave it empty, it will use memory as the context store.
+
+- **Getting Context**:
+
+When retrieving persistent context, use the `get` method with two arguments. The second argument is optional and specifies the context store (either memory or persistent).
+
+Example:
+
+```javascript
+var value = context.get('myKey', 'persistent');
+```
 
-FlowFuse provides 1MB of context storage per Node-RED instance.
+If you don't specify the store, it defaults to memory. 
 
-For more information on Context, head over to the [Node-RED Working with context](https://nodered.org/docs/user-guide/context) guide.
+For more detailed information, refer to the article [Understanding Node, Flow, Global, and Environment Variables in Node-RED](https://flowfuse.com/blog/2024/05/understanding-node-flow-global-environment-variables-in-node-red/).

forge/db/controllers/Project.js

@@ -204,7 +204,7 @@ module.exports = {
                 if (!project.ProjectTemplate) {
                     project = await app.db.models.Project.byId(project.id)
                 }
-                const newSettings = app.db.controllers.ProjectTemplate.validateSettings(snapshotSettings, project.ProjectTemplate)
+                const newSettings = app.db.controllers.ProjectTemplate.validateSettings(snapshotSettings, project.ProjectTemplate, true)
                 const currentProjectSettings = await project.getSetting('settings') || {} // necessary?
                 const updatedSettings = app.db.controllers.ProjectTemplate.mergeSettings(currentProjectSettings, newSettings, { mergeEnvVars, mergeEditorSettings, targetTemplate: project.ProjectTemplate })
                 await project.updateSetting('settings', updatedSettings, { transaction: t }) // necessary?

forge/db/controllers/ProjectTemplate.js

@@ -41,7 +41,7 @@ module.exports = {
    * @param {*} template the template to validate against
    * @returns the validated and cleansed object
    */
-    validateSettings: function (app, settings, template) {
+    validateSettings: function (app, settings, template, importing = false) {
         const result = {}
         // First pass - copy over only the known and policy-permitted settings
         templateFields.forEach((name) => {
@@ -195,7 +195,7 @@ module.exports = {
                 }
             }
         }
-        if (typeof result.httpNodeAuth?.pass === 'string' && result.httpNodeAuth.pass.length > 0) {
+        if (!importing && typeof result.httpNodeAuth?.pass === 'string' && result.httpNodeAuth.pass.length > 0) {
             result.httpNodeAuth.pass = hash(result.httpNodeAuth.pass)
         }
         if (result.apiMaxLength) {

forge/ee/routes/teamBroker/index.js

@@ -1,3 +1,5 @@
+const schemaApi = require('./schema')
+
 module.exports = async function (app) {
     app.addHook('preHandler', app.verifySession)
 
@@ -35,6 +37,8 @@ module.exports = async function (app) {
         }
     })
 
+    await schemaApi(app)
+
     /**
      * Get the Teams MQTT Clients
      * @name /api/v1/teams/:teamId/broker/clients

forge/ee/routes/teamBroker/schema.js

@@ -0,0 +1,42 @@
+const YAML = require('yaml')
+module.exports = async function (app) {
+    app.get('/team-broker/schema.yml', async (request, reply) => {
+        const list = await app.teamBroker.getUsedTopics(request.team.hashid)
+        const schema = {
+            asyncapi: '3.0.0',
+            info: {
+                version: '1.0.0',
+                title: `${request.team.name} Team Broker`,
+                description: 'An auto-generated schema of the topics being used on the team broker'
+            }
+        }
+        // Add the team-broker details
+
+        // Figure out the hostname for the team broker
+        let teamBrokerHost = app.config.broker?.teamBroker?.host
+        if (!teamBrokerHost) {
+            // No explict value set, default to broker.${domain}
+            if (app.config.domain) {
+                teamBrokerHost = `broker.${app.config.domain}`
+            }
+        }
+        if (teamBrokerHost) {
+            schema.servers = {
+                'team-broker': {
+                    host: teamBrokerHost,
+                    protocol: 'mqtt'
+                }
+            }
+        }
+
+        if (list.length > 0) {
+            schema.channels = {}
+            list.forEach(topic => {
+                schema.channels[topic] = {
+                    address: topic
+                }
+            })
+        }
+        reply.send(YAML.stringify(schema))
+    })
+}