From 466d2a6be55160cd8bd69f7f126caf20d9e67b71 Mon Sep 17 00:00:00 2001 From: Felipe Cardozo Date: Fri, 22 Nov 2024 00:42:23 -0300 Subject: [PATCH] fix jolt metions --- .../cdk-hub-download-http-source-output.bash | 5 + .../cmds/cdk-hub-download-http-source.bash | 3 +- .../cmds/hub-sm-download-jolt-output.bash | 5 + docs/_embeds/cmds/hub-sm-download-jolt.bash | 1 - .../cmds/hub-sm-download-json-sql-output.bash | 5 + docs/cloud/tutorials/mqtt-to-sql.mdx | 29 ++--- docs/fluvio/cli/fluvio/hub/list.mdx | 2 +- docs/fluvio/concepts/transformations.mdx | 10 +- docs/hub/smartmodules/jolt.mdx | 13 +- src/theme/CodeBlock/index.tsx | 11 +- .../cdk-deploy-start-http-sink-generic.bash | 1 + .../cmds/cdk-hub-download-http-sink.bash | 1 + .../cdk-hub-download-http-source-output.bash | 5 + .../cmds/cdk-hub-download-http-source.bash | 3 +- .../_embeds/cmds/hub-conn-list.bash | 13 ++ .../cmds/hub-sm-download-jolt-output.bash | 5 + .../_embeds/cmds/hub-sm-download-jolt.bash | 1 - .../cmds/hub-sm-download-json-sql-output.bash | 5 + .../_embeds/cmds/hub-sm-list.bash | 15 +++ .../cloud/tutorials/mqtt-to-sql.mdx | 31 ++--- .../fluvio/cli/fluvio/hub/list.mdx | 2 +- .../fluvio/concepts/transformations.mdx | 10 +- .../version-0.11.11/hub/smartmodules/jolt.mdx | 15 +-- .../version-0.11.11/hub/use-the-hub.mdx | 115 +++++++++++------- .../cdk-deploy-start-http-sink-generic.bash | 1 + .../cmds/cdk-hub-download-http-sink.bash | 1 + .../cdk-hub-download-http-source-output.bash | 5 + .../cmds/cdk-hub-download-http-source.bash | 3 +- .../_embeds/cmds/hub-conn-list.bash | 13 ++ .../cmds/hub-sm-download-jolt-output.bash | 5 + .../_embeds/cmds/hub-sm-download-jolt.bash | 1 - .../cmds/hub-sm-download-json-sql-output.bash | 5 + .../_embeds/cmds/hub-sm-list.bash | 15 +++ .../cloud/tutorials/mqtt-to-sql.mdx | 29 ++--- .../fluvio/cli/fluvio/hub/list.mdx | 2 +- .../fluvio/concepts/transformations.mdx | 10 +- .../version-0.11.12/hub/smartmodules/jolt.mdx | 13 +- .../version-0.11.12/hub/use-the-hub.mdx | 115 +++++++++++------- .../cdk-deploy-start-http-sink-generic.bash | 1 + .../cmds/cdk-hub-download-http-sink.bash | 1 + .../cdk-hub-download-http-source-output.bash | 5 + .../cmds/cdk-hub-download-http-source.bash | 3 +- .../_embeds/cmds/hub-conn-list.bash | 13 ++ .../cmds/hub-sm-download-jolt-output.bash | 5 + .../_embeds/cmds/hub-sm-download-jolt.bash | 1 - .../cmds/hub-sm-download-json-sql-output.bash | 5 + .../_embeds/cmds/hub-sm-list.bash | 15 +++ .../cloud/tutorials/mqtt-to-sql.mdx | 29 ++--- .../fluvio/cli/fluvio/hub/list.mdx | 2 +- .../fluvio/concepts/transformations.mdx | 10 +- .../version-0.12.0/hub/smartmodules/jolt.mdx | 13 +- .../version-0.12.0/hub/use-the-hub.mdx | 115 +++++++++++------- .../cdk-deploy-start-http-sink-generic.bash | 1 + .../cmds/cdk-hub-download-http-sink.bash | 1 + .../cdk-hub-download-http-source-output.bash | 5 + .../cmds/cdk-hub-download-http-source.bash | 3 +- .../_embeds/cmds/hub-conn-list.bash | 13 ++ .../cmds/hub-sm-download-jolt-output.bash | 5 + .../_embeds/cmds/hub-sm-download-jolt.bash | 1 - .../cmds/hub-sm-download-json-sql-output.bash | 5 + .../_embeds/cmds/hub-sm-list.bash | 15 +++ .../cloud/tutorials/mqtt-to-sql.mdx | 29 ++--- .../fluvio/cli/fluvio/hub/list.mdx | 2 +- .../fluvio/concepts/transformations.mdx | 10 +- .../version-0.13.0/hub/smartmodules/jolt.mdx | 13 +- .../version-0.13.0/hub/use-the-hub.mdx | 115 +++++++++++------- 66 files changed, 613 insertions(+), 347 deletions(-) create mode 100644 docs/_embeds/cmds/cdk-hub-download-http-source-output.bash create mode 100644 docs/_embeds/cmds/hub-sm-download-jolt-output.bash create mode 100644 docs/_embeds/cmds/hub-sm-download-json-sql-output.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-sink.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source-output.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-conn-list.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt-output.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql-output.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-list.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-sink.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source-output.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/hub-conn-list.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt-output.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-json-sql-output.bash create mode 100644 versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-list.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-sink.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source-output.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/hub-conn-list.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt-output.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-json-sql-output.bash create mode 100644 versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-list.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-sink.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source-output.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/hub-conn-list.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt-output.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-json-sql-output.bash create mode 100644 versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-list.bash diff --git a/docs/_embeds/cmds/cdk-hub-download-http-source-output.bash b/docs/_embeds/cmds/cdk-hub-download-http-source-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/docs/_embeds/cmds/cdk-hub-download-http-source-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/docs/_embeds/cmds/cdk-hub-download-http-source.bash b/docs/_embeds/cmds/cdk-hub-download-http-source.bash index 76576b6a..df74f29a 100644 --- a/docs/_embeds/cmds/cdk-hub-download-http-source.bash +++ b/docs/_embeds/cmds/cdk-hub-download-http-source.bash @@ -1 +1,2 @@ -cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file +downloading infinyon/http-source@0.3.8 to infinyon-http-source-0.3.8.ipkg +... downloading complete diff --git a/docs/_embeds/cmds/hub-sm-download-jolt-output.bash b/docs/_embeds/cmds/hub-sm-download-jolt-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/docs/_embeds/cmds/hub-sm-download-jolt-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/docs/_embeds/cmds/hub-sm-download-jolt.bash b/docs/_embeds/cmds/hub-sm-download-jolt.bash index 29b73e57..56716589 100644 --- a/docs/_embeds/cmds/hub-sm-download-jolt.bash +++ b/docs/_embeds/cmds/hub-sm-download-jolt.bash @@ -1,2 +1 @@ fluvio hub smartmodule download infinyon/jolt@0.4.1 -... cluster smartmodule install complete diff --git a/docs/_embeds/cmds/hub-sm-download-json-sql-output.bash b/docs/_embeds/cmds/hub-sm-download-json-sql-output.bash new file mode 100644 index 00000000..a7ef2fcc --- /dev/null +++ b/docs/_embeds/cmds/hub-sm-download-json-sql-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/json-sql@0.2.1 to infinyon-json-sql-0.2.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/docs/cloud/tutorials/mqtt-to-sql.mdx b/docs/cloud/tutorials/mqtt-to-sql.mdx index 6bfe829d..1f7b011c 100644 --- a/docs/cloud/tutorials/mqtt-to-sql.mdx +++ b/docs/cloud/tutorials/mqtt-to-sql.mdx @@ -137,27 +137,20 @@ $ fluvio cloud connector create --config sql.yml Download the [Jolt] and [Json-Sql] SmartModules used by this example connector -Example output - +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import SmDownloadJoltOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt-output.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmDownloadJsonSqlOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql-output.bash'; -```bash -$ fluvio hub sm download infinyon/json-sql@x.y.z -downloading infinyon/json-sql@x.y.z to infinyon-json-sql-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +$ {SmDownloadJsonSql} $ {SmDownloadJolt} +Expect the following output: -```bash -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` + + {SmDownloadJsonSqlOutput} + {SmDownloadJoltOutput} + ##### Connector with JSON to JSON transformation config diff --git a/docs/fluvio/cli/fluvio/hub/list.mdx b/docs/fluvio/cli/fluvio/hub/list.mdx index 38c7b64f..cb0d9667 100644 --- a/docs/fluvio/cli/fluvio/hub/list.mdx +++ b/docs/fluvio/cli/fluvio/hub/list.mdx @@ -18,7 +18,7 @@ Listing Hub SmartModule from the CLI is straight forward: $ fluvio hub list SMARTMODULE Visibility - infinyon/jolt@0.1.0 public + infinyon/jolt@0.4.1 public infinyon/json-sql@0.1.0 public infinyon/regex-filter@0.1.0 public ... diff --git a/docs/fluvio/concepts/transformations.mdx b/docs/fluvio/concepts/transformations.mdx index db57df5e..85e6a92b 100644 --- a/docs/fluvio/concepts/transformations.mdx +++ b/docs/fluvio/concepts/transformations.mdx @@ -22,7 +22,7 @@ Example: ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: default @@ -30,14 +30,14 @@ transforms: source: "http" ``` -we have one transformation, which is a SmartModule named [`infinyon/jolt@0.3.0`]. +we have one transformation, which is a SmartModule named [`infinyon/jolt@0.4.1`]. The name must match a SmartModule previously downloaded to the Cluster: ```bash fluvio sm list SMARTMODULE SIZE - infinyon/jolt@0.3.0 608.4 KB + infinyon/jolt@0.4.1 608.4 KB ``` @@ -93,7 +93,7 @@ Multiple SmartModules can be activated in series. The output of the previous Sma ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: shift @@ -108,4 +108,4 @@ transforms: In this example, the `jolt` transformation is performed first, then its output is the input the `regex-filter`. [SmartModules]: smartmodules/overview.mdx -[`infinyon/jolt@0.3.0`]: hub/smartmodules/jolt.mdx \ No newline at end of file +[`infinyon/jolt@0.4.1`]: hub/smartmodules/jolt.mdx \ No newline at end of file diff --git a/docs/hub/smartmodules/jolt.mdx b/docs/hub/smartmodules/jolt.mdx index abb1b1b8..ff7dd0eb 100644 --- a/docs/hub/smartmodules/jolt.mdx +++ b/docs/hub/smartmodules/jolt.mdx @@ -52,22 +52,23 @@ The Jolt SmartModule applies the operations in sequence: `remove` followed by `s To demonstrate how **Jolt** SmartModule works we will use [SMDK] tool. :::info -All smartmodule versions are marked with `x.y.z`. To find the latest version, run: +All smartmodule versions are marked with `0.4.1`. To find the latest version, run: * `fluvio hub smartmodule list` ::: First, we need to download it to our cluster: -```bash copy="fl" -$ fluvio hub smartmodule download infinyon/jolt@x.y.z -``` +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; + +$ {SmDownloadJolt} Second, we create a file `transform.yaml` with transformation specification defined above: ```yaml # transform.yaml transforms: - - uses: infinyon/jolt@x.y.z + - uses: infinyon/jolt@0.4.1 spec: - operation: remove spec: @@ -113,4 +114,4 @@ For additional examples checkout the tutorials: [Fluvio Jolt]: https://github.com/infinyon/fluvio-jolt [map-type]:smartmodules/features/operators/map.mdx -[SMDK]: smartmodules/smdk.mdx \ No newline at end of file +[SMDK]: smartmodules/smdk.mdx diff --git a/src/theme/CodeBlock/index.tsx b/src/theme/CodeBlock/index.tsx index 60cc18ed..27ec0dbc 100644 --- a/src/theme/CodeBlock/index.tsx +++ b/src/theme/CodeBlock/index.tsx @@ -17,7 +17,16 @@ function maybeStringifyChildren(children: ReactNode): ReactNode { return children; } // The children is now guaranteed to be one/more plain strings - return Array.isArray(children) ? children.join('') : (children as string); + + // If child is "$" or "$$" jump to the next line, if not add a space + return React.Children.toArray(children).map((child, index) => { + if (typeof child === 'string') { + if (index > 0) { + return child.replace(/\$\$?/g, (match) => (match === '$' ? '\n$' : ' ')); + } + } + return child; + }); } export default function CodeBlock({ diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash new file mode 100644 index 00000000..110b4f0f --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-sink-0.2.11.ipkg --config \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-sink.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-sink.bash new file mode 100644 index 00000000..32db2c56 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-sink.bash @@ -0,0 +1 @@ +cdk hub download infinyon/http-sink@0.2.11 \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source-output.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash index 76576b6a..df74f29a 100644 --- a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash @@ -1 +1,2 @@ -cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file +downloading infinyon/http-source@0.3.8 to infinyon-http-source-0.3.8.ipkg +... downloading complete diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-conn-list.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-conn-list.bash new file mode 100644 index 00000000..a7189a7d --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-conn-list.bash @@ -0,0 +1,13 @@ +fluvio hub connector list + CONNECTOR Visibility + infinyon-labs/graphite-sink@0.2.0 public + infinyon/duckdb-sink@0.1.0 public + infinyon/http-sink@0.2.11 public + infinyon/http-source@0.4.3 public + infinyon/ic-webhook-source@0.1.4 public + infinyon/inf-webhook-source@0.1.9 public + infinyon/kafka-sink@0.2.9 public + infinyon/kafka-source@0.2.7 public + infinyon/mqtt-source@0.2.9 public + infinyon/sql-sink@0.4.3 public + qdrant/qdrant-sink@0.1.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt-output.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash index 29b73e57..56716589 100644 --- a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash @@ -1,2 +1 @@ fluvio hub smartmodule download infinyon/jolt@0.4.1 -... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql-output.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql-output.bash new file mode 100644 index 00000000..a7ef2fcc --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/json-sql@0.2.1 to infinyon-json-sql-0.2.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-list.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-list.bash new file mode 100644 index 00000000..fb0b0602 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-list.bash @@ -0,0 +1,15 @@ +fluvio hub smartmodule list + SMARTMODULE Visibility + infinyon-labs/array-map-json@0.1.1 public + infinyon-labs/csv-json@0.2.0 public + infinyon-labs/dedup-filter@0.0.2 public + infinyon-labs/json-formatter@0.1.3 public + infinyon-labs/key-gen-json@0.1.1 public + infinyon-labs/regex-map-json@0.1.2 public + infinyon-labs/regex-map@0.1.1 public + infinyon-labs/rss-json@0.1.1 public + infinyon-labs/stars-forks-changes@0.1.4 public + infinyon/csv-json@0.2.0 public + infinyon/jolt@0.4.1 public + infinyon/json-sql@0.2.1 public + infinyon/regex-filter@0.2.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/cloud/tutorials/mqtt-to-sql.mdx b/versioned_docs/version-0.11.11/cloud/tutorials/mqtt-to-sql.mdx index 6bfe829d..15a8f20f 100644 --- a/versioned_docs/version-0.11.11/cloud/tutorials/mqtt-to-sql.mdx +++ b/versioned_docs/version-0.11.11/cloud/tutorials/mqtt-to-sql.mdx @@ -117,7 +117,7 @@ transforms: json-key: "payload.device.device_id" value: type: "int" - default: "0" + defau1t: "0" required: true "record": json-key: "$" @@ -137,27 +137,20 @@ $ fluvio cloud connector create --config sql.yml Download the [Jolt] and [Json-Sql] SmartModules used by this example connector -Example output +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import SmDownloadJoltOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt-output.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmDownloadJsonSqlOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql-output.bash'; +$ {SmDownloadJsonSql} $ {SmDownloadJolt} -```bash -$ fluvio hub sm download infinyon/json-sql@x.y.z -downloading infinyon/json-sql@x.y.z to infinyon-json-sql-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` - +Expect the following output: -```bash -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` + + {SmDownloadJsonSqlOutput} + {SmDownloadJoltOutput} + ##### Connector with JSON to JSON transformation config diff --git a/versioned_docs/version-0.11.11/fluvio/cli/fluvio/hub/list.mdx b/versioned_docs/version-0.11.11/fluvio/cli/fluvio/hub/list.mdx index 38c7b64f..cb0d9667 100644 --- a/versioned_docs/version-0.11.11/fluvio/cli/fluvio/hub/list.mdx +++ b/versioned_docs/version-0.11.11/fluvio/cli/fluvio/hub/list.mdx @@ -18,7 +18,7 @@ Listing Hub SmartModule from the CLI is straight forward: $ fluvio hub list SMARTMODULE Visibility - infinyon/jolt@0.1.0 public + infinyon/jolt@0.4.1 public infinyon/json-sql@0.1.0 public infinyon/regex-filter@0.1.0 public ... diff --git a/versioned_docs/version-0.11.11/fluvio/concepts/transformations.mdx b/versioned_docs/version-0.11.11/fluvio/concepts/transformations.mdx index db57df5e..85e6a92b 100644 --- a/versioned_docs/version-0.11.11/fluvio/concepts/transformations.mdx +++ b/versioned_docs/version-0.11.11/fluvio/concepts/transformations.mdx @@ -22,7 +22,7 @@ Example: ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: default @@ -30,14 +30,14 @@ transforms: source: "http" ``` -we have one transformation, which is a SmartModule named [`infinyon/jolt@0.3.0`]. +we have one transformation, which is a SmartModule named [`infinyon/jolt@0.4.1`]. The name must match a SmartModule previously downloaded to the Cluster: ```bash fluvio sm list SMARTMODULE SIZE - infinyon/jolt@0.3.0 608.4 KB + infinyon/jolt@0.4.1 608.4 KB ``` @@ -93,7 +93,7 @@ Multiple SmartModules can be activated in series. The output of the previous Sma ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: shift @@ -108,4 +108,4 @@ transforms: In this example, the `jolt` transformation is performed first, then its output is the input the `regex-filter`. [SmartModules]: smartmodules/overview.mdx -[`infinyon/jolt@0.3.0`]: hub/smartmodules/jolt.mdx \ No newline at end of file +[`infinyon/jolt@0.4.1`]: hub/smartmodules/jolt.mdx \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/hub/smartmodules/jolt.mdx b/versioned_docs/version-0.11.11/hub/smartmodules/jolt.mdx index 121d3e7e..ff7dd0eb 100644 --- a/versioned_docs/version-0.11.11/hub/smartmodules/jolt.mdx +++ b/versioned_docs/version-0.11.11/hub/smartmodules/jolt.mdx @@ -52,22 +52,23 @@ The Jolt SmartModule applies the operations in sequence: `remove` followed by `s To demonstrate how **Jolt** SmartModule works we will use [SMDK] tool. :::info -All smartmodule versions are marked with `x.y.z`. To find the latest version, run: +All smartmodule versions are marked with `0.4.1`. To find the latest version, run: * `fluvio hub smartmodule list` ::: First, we need to download it to our cluster: -```bash copy="fl" -$ fluvio hub smartmodule download infinyon/jolt@x.y.z -``` +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; + +$ {SmDownloadJolt} Second, we create a file `transform.yaml` with transformation specification defined above: ```yaml # transform.yaml transforms: - - uses: infinyon/jolt@x.y.z + - uses: infinyon/jolt@0.4.1 spec: - operation: remove spec: @@ -86,7 +87,7 @@ Let's use `smdk test` to see it in action: ```bash copy="fl" $ smdk test --text '{}' --transforms-file ./transform.yaml -1 records outputed +1 records outputted {"data":{"source":"http-connector"}} ``` @@ -113,4 +114,4 @@ For additional examples checkout the tutorials: [Fluvio Jolt]: https://github.com/infinyon/fluvio-jolt [map-type]:smartmodules/features/operators/map.mdx -[SMDK]: smartmodules/smdk.mdx \ No newline at end of file +[SMDK]: smartmodules/smdk.mdx diff --git a/versioned_docs/version-0.11.11/hub/use-the-hub.mdx b/versioned_docs/version-0.11.11/hub/use-the-hub.mdx index 0b8e1bbd..dacfe245 100644 --- a/versioned_docs/version-0.11.11/hub/use-the-hub.mdx +++ b/versioned_docs/version-0.11.11/hub/use-the-hub.mdx @@ -3,68 +3,91 @@ sidebar_position: 2 title: "Use the Hub" description: "The Hub is a central repository for Connectors, SmartModules, Dataflows, and more" --- +import CodeBlock from '@theme/CodeBlock'; +import HubConnList from '!!raw-loader!../_embeds/cmds/hub-conn-list.bash'; +import HubSmList from '!!raw-loader!../_embeds/cmds/hub-sm-list.bash'; +import HubSmDownloadJolt from '!!raw-loader!../_embeds/cmds/hub-sm-download-jolt.bash'; +import CdkDownloadHttpSink from '!!raw-loader!../_embeds/cmds/cdk-hub-download-http-sink.bash'; +import CdkDeployHttpSinkGen from '!!raw-loader!../_embeds/cmds/cdk-deploy-start-http-sink-generic.bash'; # How to use the Hub -The InfinyOn Hub serves as a centralized repository for InfinyOn tConnectors, SmartModules, and Dataflows. It facilitates the discovery and deployment of extensions to enhance the data streaming capabilities of Fluvio. The hub provides a platform for developers and data engineers to easily access pre-built solutions or share their own extensions. +The InfinyOn Hub serves as a centralized repository for InfinyOn [Connectors](#connectors), [SmartModules](#smartmodules), and Dataflows. It facilitates the discovery and deployment of extensions to enhance Fluvio's data streaming capabilities. The hub provides developers and data engineers a platform to access pre-built solutions or share their extensions easily. -## List +## Connectors -List the SmartModules available for download. +List the Connectors available for download. The official connectors (prepended with `infinyon/`) can run on InfinyOn Cloud and in your local environment. -```shell copy="fl" -$ fluvio hub smartmodule list - SMARTMODULE Visibility - infinyon-labs/array-map-json@x.y.z public - infinyon-labs/dedup-filter@x.y.z public - infinyon-labs/json-formatter@x.y.z public - infinyon-labs/key-gen-json@x.y.z public - infinyon-labs/regex-map-json@x.y.z public - infinyon-labs/regex-map@x.y.z public - infinyon-labs/stars-forks-changes@x.y.z public - infinyon/jolt@x.y.z public - infinyon/json-sql@x.y.z public - infinyon/regex-filter@x.y.z public -``` +### List Connectors -List the Connectors available for download. +Use `fluvio hub connector list` or `cdk hub list` to list the available connectors. -```shell copy="fl" -$ fluvio hub connector list - CONNECTOR Visibility - infinyon-labs/graphite-sink@x.y.z public - infinyon/duckdb-sink@x.y.z public - infinyon/http-sink@x.y.z public - infinyon/http-source@x.y.z public - infinyon/ic-webhook-source@x.y.z public - infinyon/kafka-sink@x.y.z public - infinyon/kafka-source@x.y.z public - infinyon/mqtt-source@x.y.z public - infinyon/sql-sink@x.y.z public -``` +$ {HubConnList} +Check out [Connectors for Developers] to learn how to build and publish your own connectors for the Hub. -## Download +### Use Connectors -Downloading a SmartModule to a target cluster defined by your [`profile`]. SmartModules need to be downloaded before they are used in transformations. +If you use a connector in InfinyOn Cloud, the Cloud infrastructure will automatically download and run the connector package. If you use connectors in your local environment, you must use the `cdk` command to download and run the connector package. -```shell copy="fl" -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +#### Download and Run Locally + +Connectors are independent binaries that run outside of Fluvio. Use the `cdk` command to download and run connectors locally: + +$ {CdkDownloadHttpSink} + +The same connector configuration file can run the connector locally or in InfinyOn Cloud. + +$ {CdkDeployHttpSinkGen} + +To stop a connector, use the `cdk deploy shutdown` command. + + +#### Deploy on InfinyOn Cloud -Running `fluvio hub conn download` will download the package containing source code of the Connector. +Connectors are automatically downloaded to InfinyOn Cloud. The management of InfinyOn Cloud is performed using the `fluvio cloud` commands. To run a connector in InfinyOn Cloud, use the command `fluvio cloud connector create`. ```shell copy="fl" -$ fluvio hub conn download infinyon/http-source@x.y.z -downloading infinyon/http-source@x.y.z to infinyon-http-source-x.y.z.ipkg -... downloading complete +$ fluvio cloud connector create --config ``` +**Note** List your profiles (`fluvio profile list`) to ensure your CLI points to the InfinyOn Cloud cluster. - [`profile`]: fluvio/cli/fluvio/profile.mdx \ No newline at end of file + +## SmartModules + +SmartModules are custom-defined programs that connectors use to transform data records during processing. Similar to connectors, SmartModules can be published to the Hub and made available for download. However, unlike connectors, SmartModules are downloaded to the cluster and are not executed outside of Fluvio. + +### List SmartModules + +List the SmartModules available for download. The official SmartModules (prepended with `infinyon/`). + +$ {HubSmList} + + +### Download and Run SmartModules + +Use `fluvio hub smartmodule download` to download specific SmartModules to a local or InfinyOn Cloud cluster. Use `fluvio profile list` to identify the target cluster. + +$ {HubSmDownloadJolt} + +SmartModules downloaded to a cluster can be used by the producer CLI or consumer CLI to transform records. They can also be accessed within the Connectors via the configuration file. + +To see the SmartModules currently installed on the cluster, use the following command: + +```shell copy="fl" +$ fluvio smartmodule list + ``` + +### References +- [Connector Developer Kit (cdk)] +- [Connector Configuration File] +- [SmartModule Developer Kit (smdk)] + +[cdk deploy shutdown]: /connectors/developers/overview.mdx#shutdown-a-connector +[`profile`]: fluvio/cli/fluvio/profile.mdx +[Connectors for Developers]: /connectors/developers/overview.mdx +[Connector Configuration File]: connectors/configuration.mdx +[Connector Developer Kit (cdk)]: /connectors/developers/overview.mdx +[SmartModule Developer Kit (smdk)]: smartmodules/developers/overview.mdx diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash new file mode 100644 index 00000000..110b4f0f --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-sink-0.2.11.ipkg --config \ No newline at end of file diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-sink.bash b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-sink.bash new file mode 100644 index 00000000..32db2c56 --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-sink.bash @@ -0,0 +1 @@ +cdk hub download infinyon/http-sink@0.2.11 \ No newline at end of file diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source-output.bash b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source.bash b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source.bash index 76576b6a..df74f29a 100644 --- a/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source.bash +++ b/versioned_docs/version-0.11.12/_embeds/cmds/cdk-hub-download-http-source.bash @@ -1 +1,2 @@ -cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file +downloading infinyon/http-source@0.3.8 to infinyon-http-source-0.3.8.ipkg +... downloading complete diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/hub-conn-list.bash b/versioned_docs/version-0.11.12/_embeds/cmds/hub-conn-list.bash new file mode 100644 index 00000000..a7189a7d --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/hub-conn-list.bash @@ -0,0 +1,13 @@ +fluvio hub connector list + CONNECTOR Visibility + infinyon-labs/graphite-sink@0.2.0 public + infinyon/duckdb-sink@0.1.0 public + infinyon/http-sink@0.2.11 public + infinyon/http-source@0.4.3 public + infinyon/ic-webhook-source@0.1.4 public + infinyon/inf-webhook-source@0.1.9 public + infinyon/kafka-sink@0.2.9 public + infinyon/kafka-source@0.2.7 public + infinyon/mqtt-source@0.2.9 public + infinyon/sql-sink@0.4.3 public + qdrant/qdrant-sink@0.1.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt-output.bash b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt.bash b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt.bash index 29b73e57..56716589 100644 --- a/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt.bash +++ b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-jolt.bash @@ -1,2 +1 @@ fluvio hub smartmodule download infinyon/jolt@0.4.1 -... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-json-sql-output.bash b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-json-sql-output.bash new file mode 100644 index 00000000..a7ef2fcc --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-download-json-sql-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/json-sql@0.2.1 to infinyon-json-sql-0.2.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-list.bash b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-list.bash new file mode 100644 index 00000000..fb0b0602 --- /dev/null +++ b/versioned_docs/version-0.11.12/_embeds/cmds/hub-sm-list.bash @@ -0,0 +1,15 @@ +fluvio hub smartmodule list + SMARTMODULE Visibility + infinyon-labs/array-map-json@0.1.1 public + infinyon-labs/csv-json@0.2.0 public + infinyon-labs/dedup-filter@0.0.2 public + infinyon-labs/json-formatter@0.1.3 public + infinyon-labs/key-gen-json@0.1.1 public + infinyon-labs/regex-map-json@0.1.2 public + infinyon-labs/regex-map@0.1.1 public + infinyon-labs/rss-json@0.1.1 public + infinyon-labs/stars-forks-changes@0.1.4 public + infinyon/csv-json@0.2.0 public + infinyon/jolt@0.4.1 public + infinyon/json-sql@0.2.1 public + infinyon/regex-filter@0.2.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.11.12/cloud/tutorials/mqtt-to-sql.mdx b/versioned_docs/version-0.11.12/cloud/tutorials/mqtt-to-sql.mdx index 6bfe829d..1f7b011c 100644 --- a/versioned_docs/version-0.11.12/cloud/tutorials/mqtt-to-sql.mdx +++ b/versioned_docs/version-0.11.12/cloud/tutorials/mqtt-to-sql.mdx @@ -137,27 +137,20 @@ $ fluvio cloud connector create --config sql.yml Download the [Jolt] and [Json-Sql] SmartModules used by this example connector -Example output - +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import SmDownloadJoltOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt-output.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmDownloadJsonSqlOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql-output.bash'; -```bash -$ fluvio hub sm download infinyon/json-sql@x.y.z -downloading infinyon/json-sql@x.y.z to infinyon-json-sql-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +$ {SmDownloadJsonSql} $ {SmDownloadJolt} +Expect the following output: -```bash -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` + + {SmDownloadJsonSqlOutput} + {SmDownloadJoltOutput} + ##### Connector with JSON to JSON transformation config diff --git a/versioned_docs/version-0.11.12/fluvio/cli/fluvio/hub/list.mdx b/versioned_docs/version-0.11.12/fluvio/cli/fluvio/hub/list.mdx index 38c7b64f..cb0d9667 100644 --- a/versioned_docs/version-0.11.12/fluvio/cli/fluvio/hub/list.mdx +++ b/versioned_docs/version-0.11.12/fluvio/cli/fluvio/hub/list.mdx @@ -18,7 +18,7 @@ Listing Hub SmartModule from the CLI is straight forward: $ fluvio hub list SMARTMODULE Visibility - infinyon/jolt@0.1.0 public + infinyon/jolt@0.4.1 public infinyon/json-sql@0.1.0 public infinyon/regex-filter@0.1.0 public ... diff --git a/versioned_docs/version-0.11.12/fluvio/concepts/transformations.mdx b/versioned_docs/version-0.11.12/fluvio/concepts/transformations.mdx index db57df5e..85e6a92b 100644 --- a/versioned_docs/version-0.11.12/fluvio/concepts/transformations.mdx +++ b/versioned_docs/version-0.11.12/fluvio/concepts/transformations.mdx @@ -22,7 +22,7 @@ Example: ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: default @@ -30,14 +30,14 @@ transforms: source: "http" ``` -we have one transformation, which is a SmartModule named [`infinyon/jolt@0.3.0`]. +we have one transformation, which is a SmartModule named [`infinyon/jolt@0.4.1`]. The name must match a SmartModule previously downloaded to the Cluster: ```bash fluvio sm list SMARTMODULE SIZE - infinyon/jolt@0.3.0 608.4 KB + infinyon/jolt@0.4.1 608.4 KB ``` @@ -93,7 +93,7 @@ Multiple SmartModules can be activated in series. The output of the previous Sma ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: shift @@ -108,4 +108,4 @@ transforms: In this example, the `jolt` transformation is performed first, then its output is the input the `regex-filter`. [SmartModules]: smartmodules/overview.mdx -[`infinyon/jolt@0.3.0`]: hub/smartmodules/jolt.mdx \ No newline at end of file +[`infinyon/jolt@0.4.1`]: hub/smartmodules/jolt.mdx \ No newline at end of file diff --git a/versioned_docs/version-0.11.12/hub/smartmodules/jolt.mdx b/versioned_docs/version-0.11.12/hub/smartmodules/jolt.mdx index abb1b1b8..ff7dd0eb 100644 --- a/versioned_docs/version-0.11.12/hub/smartmodules/jolt.mdx +++ b/versioned_docs/version-0.11.12/hub/smartmodules/jolt.mdx @@ -52,22 +52,23 @@ The Jolt SmartModule applies the operations in sequence: `remove` followed by `s To demonstrate how **Jolt** SmartModule works we will use [SMDK] tool. :::info -All smartmodule versions are marked with `x.y.z`. To find the latest version, run: +All smartmodule versions are marked with `0.4.1`. To find the latest version, run: * `fluvio hub smartmodule list` ::: First, we need to download it to our cluster: -```bash copy="fl" -$ fluvio hub smartmodule download infinyon/jolt@x.y.z -``` +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; + +$ {SmDownloadJolt} Second, we create a file `transform.yaml` with transformation specification defined above: ```yaml # transform.yaml transforms: - - uses: infinyon/jolt@x.y.z + - uses: infinyon/jolt@0.4.1 spec: - operation: remove spec: @@ -113,4 +114,4 @@ For additional examples checkout the tutorials: [Fluvio Jolt]: https://github.com/infinyon/fluvio-jolt [map-type]:smartmodules/features/operators/map.mdx -[SMDK]: smartmodules/smdk.mdx \ No newline at end of file +[SMDK]: smartmodules/smdk.mdx diff --git a/versioned_docs/version-0.11.12/hub/use-the-hub.mdx b/versioned_docs/version-0.11.12/hub/use-the-hub.mdx index 0b8e1bbd..dacfe245 100644 --- a/versioned_docs/version-0.11.12/hub/use-the-hub.mdx +++ b/versioned_docs/version-0.11.12/hub/use-the-hub.mdx @@ -3,68 +3,91 @@ sidebar_position: 2 title: "Use the Hub" description: "The Hub is a central repository for Connectors, SmartModules, Dataflows, and more" --- +import CodeBlock from '@theme/CodeBlock'; +import HubConnList from '!!raw-loader!../_embeds/cmds/hub-conn-list.bash'; +import HubSmList from '!!raw-loader!../_embeds/cmds/hub-sm-list.bash'; +import HubSmDownloadJolt from '!!raw-loader!../_embeds/cmds/hub-sm-download-jolt.bash'; +import CdkDownloadHttpSink from '!!raw-loader!../_embeds/cmds/cdk-hub-download-http-sink.bash'; +import CdkDeployHttpSinkGen from '!!raw-loader!../_embeds/cmds/cdk-deploy-start-http-sink-generic.bash'; # How to use the Hub -The InfinyOn Hub serves as a centralized repository for InfinyOn tConnectors, SmartModules, and Dataflows. It facilitates the discovery and deployment of extensions to enhance the data streaming capabilities of Fluvio. The hub provides a platform for developers and data engineers to easily access pre-built solutions or share their own extensions. +The InfinyOn Hub serves as a centralized repository for InfinyOn [Connectors](#connectors), [SmartModules](#smartmodules), and Dataflows. It facilitates the discovery and deployment of extensions to enhance Fluvio's data streaming capabilities. The hub provides developers and data engineers a platform to access pre-built solutions or share their extensions easily. -## List +## Connectors -List the SmartModules available for download. +List the Connectors available for download. The official connectors (prepended with `infinyon/`) can run on InfinyOn Cloud and in your local environment. -```shell copy="fl" -$ fluvio hub smartmodule list - SMARTMODULE Visibility - infinyon-labs/array-map-json@x.y.z public - infinyon-labs/dedup-filter@x.y.z public - infinyon-labs/json-formatter@x.y.z public - infinyon-labs/key-gen-json@x.y.z public - infinyon-labs/regex-map-json@x.y.z public - infinyon-labs/regex-map@x.y.z public - infinyon-labs/stars-forks-changes@x.y.z public - infinyon/jolt@x.y.z public - infinyon/json-sql@x.y.z public - infinyon/regex-filter@x.y.z public -``` +### List Connectors -List the Connectors available for download. +Use `fluvio hub connector list` or `cdk hub list` to list the available connectors. -```shell copy="fl" -$ fluvio hub connector list - CONNECTOR Visibility - infinyon-labs/graphite-sink@x.y.z public - infinyon/duckdb-sink@x.y.z public - infinyon/http-sink@x.y.z public - infinyon/http-source@x.y.z public - infinyon/ic-webhook-source@x.y.z public - infinyon/kafka-sink@x.y.z public - infinyon/kafka-source@x.y.z public - infinyon/mqtt-source@x.y.z public - infinyon/sql-sink@x.y.z public -``` +$ {HubConnList} +Check out [Connectors for Developers] to learn how to build and publish your own connectors for the Hub. -## Download +### Use Connectors -Downloading a SmartModule to a target cluster defined by your [`profile`]. SmartModules need to be downloaded before they are used in transformations. +If you use a connector in InfinyOn Cloud, the Cloud infrastructure will automatically download and run the connector package. If you use connectors in your local environment, you must use the `cdk` command to download and run the connector package. -```shell copy="fl" -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +#### Download and Run Locally + +Connectors are independent binaries that run outside of Fluvio. Use the `cdk` command to download and run connectors locally: + +$ {CdkDownloadHttpSink} + +The same connector configuration file can run the connector locally or in InfinyOn Cloud. + +$ {CdkDeployHttpSinkGen} + +To stop a connector, use the `cdk deploy shutdown` command. + + +#### Deploy on InfinyOn Cloud -Running `fluvio hub conn download` will download the package containing source code of the Connector. +Connectors are automatically downloaded to InfinyOn Cloud. The management of InfinyOn Cloud is performed using the `fluvio cloud` commands. To run a connector in InfinyOn Cloud, use the command `fluvio cloud connector create`. ```shell copy="fl" -$ fluvio hub conn download infinyon/http-source@x.y.z -downloading infinyon/http-source@x.y.z to infinyon-http-source-x.y.z.ipkg -... downloading complete +$ fluvio cloud connector create --config ``` +**Note** List your profiles (`fluvio profile list`) to ensure your CLI points to the InfinyOn Cloud cluster. - [`profile`]: fluvio/cli/fluvio/profile.mdx \ No newline at end of file + +## SmartModules + +SmartModules are custom-defined programs that connectors use to transform data records during processing. Similar to connectors, SmartModules can be published to the Hub and made available for download. However, unlike connectors, SmartModules are downloaded to the cluster and are not executed outside of Fluvio. + +### List SmartModules + +List the SmartModules available for download. The official SmartModules (prepended with `infinyon/`). + +$ {HubSmList} + + +### Download and Run SmartModules + +Use `fluvio hub smartmodule download` to download specific SmartModules to a local or InfinyOn Cloud cluster. Use `fluvio profile list` to identify the target cluster. + +$ {HubSmDownloadJolt} + +SmartModules downloaded to a cluster can be used by the producer CLI or consumer CLI to transform records. They can also be accessed within the Connectors via the configuration file. + +To see the SmartModules currently installed on the cluster, use the following command: + +```shell copy="fl" +$ fluvio smartmodule list + ``` + +### References +- [Connector Developer Kit (cdk)] +- [Connector Configuration File] +- [SmartModule Developer Kit (smdk)] + +[cdk deploy shutdown]: /connectors/developers/overview.mdx#shutdown-a-connector +[`profile`]: fluvio/cli/fluvio/profile.mdx +[Connectors for Developers]: /connectors/developers/overview.mdx +[Connector Configuration File]: connectors/configuration.mdx +[Connector Developer Kit (cdk)]: /connectors/developers/overview.mdx +[SmartModule Developer Kit (smdk)]: smartmodules/developers/overview.mdx diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash new file mode 100644 index 00000000..110b4f0f --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-sink-0.2.11.ipkg --config \ No newline at end of file diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-sink.bash b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-sink.bash new file mode 100644 index 00000000..32db2c56 --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-sink.bash @@ -0,0 +1 @@ +cdk hub download infinyon/http-sink@0.2.11 \ No newline at end of file diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source-output.bash b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source.bash b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source.bash index 76576b6a..df74f29a 100644 --- a/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source.bash +++ b/versioned_docs/version-0.12.0/_embeds/cmds/cdk-hub-download-http-source.bash @@ -1 +1,2 @@ -cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file +downloading infinyon/http-source@0.3.8 to infinyon-http-source-0.3.8.ipkg +... downloading complete diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/hub-conn-list.bash b/versioned_docs/version-0.12.0/_embeds/cmds/hub-conn-list.bash new file mode 100644 index 00000000..a7189a7d --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/hub-conn-list.bash @@ -0,0 +1,13 @@ +fluvio hub connector list + CONNECTOR Visibility + infinyon-labs/graphite-sink@0.2.0 public + infinyon/duckdb-sink@0.1.0 public + infinyon/http-sink@0.2.11 public + infinyon/http-source@0.4.3 public + infinyon/ic-webhook-source@0.1.4 public + infinyon/inf-webhook-source@0.1.9 public + infinyon/kafka-sink@0.2.9 public + infinyon/kafka-source@0.2.7 public + infinyon/mqtt-source@0.2.9 public + infinyon/sql-sink@0.4.3 public + qdrant/qdrant-sink@0.1.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt-output.bash b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt.bash b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt.bash index 29b73e57..56716589 100644 --- a/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt.bash +++ b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-jolt.bash @@ -1,2 +1 @@ fluvio hub smartmodule download infinyon/jolt@0.4.1 -... cluster smartmodule install complete diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-json-sql-output.bash b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-json-sql-output.bash new file mode 100644 index 00000000..a7ef2fcc --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-download-json-sql-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/json-sql@0.2.1 to infinyon-json-sql-0.2.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-list.bash b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-list.bash new file mode 100644 index 00000000..fb0b0602 --- /dev/null +++ b/versioned_docs/version-0.12.0/_embeds/cmds/hub-sm-list.bash @@ -0,0 +1,15 @@ +fluvio hub smartmodule list + SMARTMODULE Visibility + infinyon-labs/array-map-json@0.1.1 public + infinyon-labs/csv-json@0.2.0 public + infinyon-labs/dedup-filter@0.0.2 public + infinyon-labs/json-formatter@0.1.3 public + infinyon-labs/key-gen-json@0.1.1 public + infinyon-labs/regex-map-json@0.1.2 public + infinyon-labs/regex-map@0.1.1 public + infinyon-labs/rss-json@0.1.1 public + infinyon-labs/stars-forks-changes@0.1.4 public + infinyon/csv-json@0.2.0 public + infinyon/jolt@0.4.1 public + infinyon/json-sql@0.2.1 public + infinyon/regex-filter@0.2.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.12.0/cloud/tutorials/mqtt-to-sql.mdx b/versioned_docs/version-0.12.0/cloud/tutorials/mqtt-to-sql.mdx index 6bfe829d..1f7b011c 100644 --- a/versioned_docs/version-0.12.0/cloud/tutorials/mqtt-to-sql.mdx +++ b/versioned_docs/version-0.12.0/cloud/tutorials/mqtt-to-sql.mdx @@ -137,27 +137,20 @@ $ fluvio cloud connector create --config sql.yml Download the [Jolt] and [Json-Sql] SmartModules used by this example connector -Example output - +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import SmDownloadJoltOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt-output.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmDownloadJsonSqlOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql-output.bash'; -```bash -$ fluvio hub sm download infinyon/json-sql@x.y.z -downloading infinyon/json-sql@x.y.z to infinyon-json-sql-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +$ {SmDownloadJsonSql} $ {SmDownloadJolt} +Expect the following output: -```bash -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` + + {SmDownloadJsonSqlOutput} + {SmDownloadJoltOutput} + ##### Connector with JSON to JSON transformation config diff --git a/versioned_docs/version-0.12.0/fluvio/cli/fluvio/hub/list.mdx b/versioned_docs/version-0.12.0/fluvio/cli/fluvio/hub/list.mdx index 38c7b64f..cb0d9667 100644 --- a/versioned_docs/version-0.12.0/fluvio/cli/fluvio/hub/list.mdx +++ b/versioned_docs/version-0.12.0/fluvio/cli/fluvio/hub/list.mdx @@ -18,7 +18,7 @@ Listing Hub SmartModule from the CLI is straight forward: $ fluvio hub list SMARTMODULE Visibility - infinyon/jolt@0.1.0 public + infinyon/jolt@0.4.1 public infinyon/json-sql@0.1.0 public infinyon/regex-filter@0.1.0 public ... diff --git a/versioned_docs/version-0.12.0/fluvio/concepts/transformations.mdx b/versioned_docs/version-0.12.0/fluvio/concepts/transformations.mdx index db57df5e..85e6a92b 100644 --- a/versioned_docs/version-0.12.0/fluvio/concepts/transformations.mdx +++ b/versioned_docs/version-0.12.0/fluvio/concepts/transformations.mdx @@ -22,7 +22,7 @@ Example: ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: default @@ -30,14 +30,14 @@ transforms: source: "http" ``` -we have one transformation, which is a SmartModule named [`infinyon/jolt@0.3.0`]. +we have one transformation, which is a SmartModule named [`infinyon/jolt@0.4.1`]. The name must match a SmartModule previously downloaded to the Cluster: ```bash fluvio sm list SMARTMODULE SIZE - infinyon/jolt@0.3.0 608.4 KB + infinyon/jolt@0.4.1 608.4 KB ``` @@ -93,7 +93,7 @@ Multiple SmartModules can be activated in series. The output of the previous Sma ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: shift @@ -108,4 +108,4 @@ transforms: In this example, the `jolt` transformation is performed first, then its output is the input the `regex-filter`. [SmartModules]: smartmodules/overview.mdx -[`infinyon/jolt@0.3.0`]: hub/smartmodules/jolt.mdx \ No newline at end of file +[`infinyon/jolt@0.4.1`]: hub/smartmodules/jolt.mdx \ No newline at end of file diff --git a/versioned_docs/version-0.12.0/hub/smartmodules/jolt.mdx b/versioned_docs/version-0.12.0/hub/smartmodules/jolt.mdx index abb1b1b8..ff7dd0eb 100644 --- a/versioned_docs/version-0.12.0/hub/smartmodules/jolt.mdx +++ b/versioned_docs/version-0.12.0/hub/smartmodules/jolt.mdx @@ -52,22 +52,23 @@ The Jolt SmartModule applies the operations in sequence: `remove` followed by `s To demonstrate how **Jolt** SmartModule works we will use [SMDK] tool. :::info -All smartmodule versions are marked with `x.y.z`. To find the latest version, run: +All smartmodule versions are marked with `0.4.1`. To find the latest version, run: * `fluvio hub smartmodule list` ::: First, we need to download it to our cluster: -```bash copy="fl" -$ fluvio hub smartmodule download infinyon/jolt@x.y.z -``` +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; + +$ {SmDownloadJolt} Second, we create a file `transform.yaml` with transformation specification defined above: ```yaml # transform.yaml transforms: - - uses: infinyon/jolt@x.y.z + - uses: infinyon/jolt@0.4.1 spec: - operation: remove spec: @@ -113,4 +114,4 @@ For additional examples checkout the tutorials: [Fluvio Jolt]: https://github.com/infinyon/fluvio-jolt [map-type]:smartmodules/features/operators/map.mdx -[SMDK]: smartmodules/smdk.mdx \ No newline at end of file +[SMDK]: smartmodules/smdk.mdx diff --git a/versioned_docs/version-0.12.0/hub/use-the-hub.mdx b/versioned_docs/version-0.12.0/hub/use-the-hub.mdx index 0b8e1bbd..dacfe245 100644 --- a/versioned_docs/version-0.12.0/hub/use-the-hub.mdx +++ b/versioned_docs/version-0.12.0/hub/use-the-hub.mdx @@ -3,68 +3,91 @@ sidebar_position: 2 title: "Use the Hub" description: "The Hub is a central repository for Connectors, SmartModules, Dataflows, and more" --- +import CodeBlock from '@theme/CodeBlock'; +import HubConnList from '!!raw-loader!../_embeds/cmds/hub-conn-list.bash'; +import HubSmList from '!!raw-loader!../_embeds/cmds/hub-sm-list.bash'; +import HubSmDownloadJolt from '!!raw-loader!../_embeds/cmds/hub-sm-download-jolt.bash'; +import CdkDownloadHttpSink from '!!raw-loader!../_embeds/cmds/cdk-hub-download-http-sink.bash'; +import CdkDeployHttpSinkGen from '!!raw-loader!../_embeds/cmds/cdk-deploy-start-http-sink-generic.bash'; # How to use the Hub -The InfinyOn Hub serves as a centralized repository for InfinyOn tConnectors, SmartModules, and Dataflows. It facilitates the discovery and deployment of extensions to enhance the data streaming capabilities of Fluvio. The hub provides a platform for developers and data engineers to easily access pre-built solutions or share their own extensions. +The InfinyOn Hub serves as a centralized repository for InfinyOn [Connectors](#connectors), [SmartModules](#smartmodules), and Dataflows. It facilitates the discovery and deployment of extensions to enhance Fluvio's data streaming capabilities. The hub provides developers and data engineers a platform to access pre-built solutions or share their extensions easily. -## List +## Connectors -List the SmartModules available for download. +List the Connectors available for download. The official connectors (prepended with `infinyon/`) can run on InfinyOn Cloud and in your local environment. -```shell copy="fl" -$ fluvio hub smartmodule list - SMARTMODULE Visibility - infinyon-labs/array-map-json@x.y.z public - infinyon-labs/dedup-filter@x.y.z public - infinyon-labs/json-formatter@x.y.z public - infinyon-labs/key-gen-json@x.y.z public - infinyon-labs/regex-map-json@x.y.z public - infinyon-labs/regex-map@x.y.z public - infinyon-labs/stars-forks-changes@x.y.z public - infinyon/jolt@x.y.z public - infinyon/json-sql@x.y.z public - infinyon/regex-filter@x.y.z public -``` +### List Connectors -List the Connectors available for download. +Use `fluvio hub connector list` or `cdk hub list` to list the available connectors. -```shell copy="fl" -$ fluvio hub connector list - CONNECTOR Visibility - infinyon-labs/graphite-sink@x.y.z public - infinyon/duckdb-sink@x.y.z public - infinyon/http-sink@x.y.z public - infinyon/http-source@x.y.z public - infinyon/ic-webhook-source@x.y.z public - infinyon/kafka-sink@x.y.z public - infinyon/kafka-source@x.y.z public - infinyon/mqtt-source@x.y.z public - infinyon/sql-sink@x.y.z public -``` +$ {HubConnList} +Check out [Connectors for Developers] to learn how to build and publish your own connectors for the Hub. -## Download +### Use Connectors -Downloading a SmartModule to a target cluster defined by your [`profile`]. SmartModules need to be downloaded before they are used in transformations. +If you use a connector in InfinyOn Cloud, the Cloud infrastructure will automatically download and run the connector package. If you use connectors in your local environment, you must use the `cdk` command to download and run the connector package. -```shell copy="fl" -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +#### Download and Run Locally + +Connectors are independent binaries that run outside of Fluvio. Use the `cdk` command to download and run connectors locally: + +$ {CdkDownloadHttpSink} + +The same connector configuration file can run the connector locally or in InfinyOn Cloud. + +$ {CdkDeployHttpSinkGen} + +To stop a connector, use the `cdk deploy shutdown` command. + + +#### Deploy on InfinyOn Cloud -Running `fluvio hub conn download` will download the package containing source code of the Connector. +Connectors are automatically downloaded to InfinyOn Cloud. The management of InfinyOn Cloud is performed using the `fluvio cloud` commands. To run a connector in InfinyOn Cloud, use the command `fluvio cloud connector create`. ```shell copy="fl" -$ fluvio hub conn download infinyon/http-source@x.y.z -downloading infinyon/http-source@x.y.z to infinyon-http-source-x.y.z.ipkg -... downloading complete +$ fluvio cloud connector create --config ``` +**Note** List your profiles (`fluvio profile list`) to ensure your CLI points to the InfinyOn Cloud cluster. - [`profile`]: fluvio/cli/fluvio/profile.mdx \ No newline at end of file + +## SmartModules + +SmartModules are custom-defined programs that connectors use to transform data records during processing. Similar to connectors, SmartModules can be published to the Hub and made available for download. However, unlike connectors, SmartModules are downloaded to the cluster and are not executed outside of Fluvio. + +### List SmartModules + +List the SmartModules available for download. The official SmartModules (prepended with `infinyon/`). + +$ {HubSmList} + + +### Download and Run SmartModules + +Use `fluvio hub smartmodule download` to download specific SmartModules to a local or InfinyOn Cloud cluster. Use `fluvio profile list` to identify the target cluster. + +$ {HubSmDownloadJolt} + +SmartModules downloaded to a cluster can be used by the producer CLI or consumer CLI to transform records. They can also be accessed within the Connectors via the configuration file. + +To see the SmartModules currently installed on the cluster, use the following command: + +```shell copy="fl" +$ fluvio smartmodule list + ``` + +### References +- [Connector Developer Kit (cdk)] +- [Connector Configuration File] +- [SmartModule Developer Kit (smdk)] + +[cdk deploy shutdown]: /connectors/developers/overview.mdx#shutdown-a-connector +[`profile`]: fluvio/cli/fluvio/profile.mdx +[Connectors for Developers]: /connectors/developers/overview.mdx +[Connector Configuration File]: connectors/configuration.mdx +[Connector Developer Kit (cdk)]: /connectors/developers/overview.mdx +[SmartModule Developer Kit (smdk)]: smartmodules/developers/overview.mdx diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash new file mode 100644 index 00000000..110b4f0f --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-deploy-start-http-sink-generic.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-sink-0.2.11.ipkg --config \ No newline at end of file diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-sink.bash b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-sink.bash new file mode 100644 index 00000000..32db2c56 --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-sink.bash @@ -0,0 +1 @@ +cdk hub download infinyon/http-sink@0.2.11 \ No newline at end of file diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source-output.bash b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source.bash b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source.bash index 76576b6a..df74f29a 100644 --- a/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source.bash +++ b/versioned_docs/version-0.13.0/_embeds/cmds/cdk-hub-download-http-source.bash @@ -1 +1,2 @@ -cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file +downloading infinyon/http-source@0.3.8 to infinyon-http-source-0.3.8.ipkg +... downloading complete diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/hub-conn-list.bash b/versioned_docs/version-0.13.0/_embeds/cmds/hub-conn-list.bash new file mode 100644 index 00000000..a7189a7d --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/hub-conn-list.bash @@ -0,0 +1,13 @@ +fluvio hub connector list + CONNECTOR Visibility + infinyon-labs/graphite-sink@0.2.0 public + infinyon/duckdb-sink@0.1.0 public + infinyon/http-sink@0.2.11 public + infinyon/http-source@0.4.3 public + infinyon/ic-webhook-source@0.1.4 public + infinyon/inf-webhook-source@0.1.9 public + infinyon/kafka-sink@0.2.9 public + infinyon/kafka-source@0.2.7 public + infinyon/mqtt-source@0.2.9 public + infinyon/sql-sink@0.4.3 public + qdrant/qdrant-sink@0.1.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt-output.bash b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt-output.bash new file mode 100644 index 00000000..c09b7aa7 --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/jolt@0.4.1 to infinyon-jolt-0.4.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt.bash b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt.bash index 29b73e57..56716589 100644 --- a/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt.bash +++ b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-jolt.bash @@ -1,2 +1 @@ fluvio hub smartmodule download infinyon/jolt@0.4.1 -... cluster smartmodule install complete diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-json-sql-output.bash b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-json-sql-output.bash new file mode 100644 index 00000000..a7ef2fcc --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-download-json-sql-output.bash @@ -0,0 +1,5 @@ +trying connection to fluvio router.infinyon.cloud:9003 +downloading infinyon/json-sql@0.2.1 to infinyon-json-sql-0.2.1.ipkg +... downloading complete +... checking package +... cluster smartmodule install complete diff --git a/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-list.bash b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-list.bash new file mode 100644 index 00000000..fb0b0602 --- /dev/null +++ b/versioned_docs/version-0.13.0/_embeds/cmds/hub-sm-list.bash @@ -0,0 +1,15 @@ +fluvio hub smartmodule list + SMARTMODULE Visibility + infinyon-labs/array-map-json@0.1.1 public + infinyon-labs/csv-json@0.2.0 public + infinyon-labs/dedup-filter@0.0.2 public + infinyon-labs/json-formatter@0.1.3 public + infinyon-labs/key-gen-json@0.1.1 public + infinyon-labs/regex-map-json@0.1.2 public + infinyon-labs/regex-map@0.1.1 public + infinyon-labs/rss-json@0.1.1 public + infinyon-labs/stars-forks-changes@0.1.4 public + infinyon/csv-json@0.2.0 public + infinyon/jolt@0.4.1 public + infinyon/json-sql@0.2.1 public + infinyon/regex-filter@0.2.0 public \ No newline at end of file diff --git a/versioned_docs/version-0.13.0/cloud/tutorials/mqtt-to-sql.mdx b/versioned_docs/version-0.13.0/cloud/tutorials/mqtt-to-sql.mdx index 6bfe829d..1f7b011c 100644 --- a/versioned_docs/version-0.13.0/cloud/tutorials/mqtt-to-sql.mdx +++ b/versioned_docs/version-0.13.0/cloud/tutorials/mqtt-to-sql.mdx @@ -137,27 +137,20 @@ $ fluvio cloud connector create --config sql.yml Download the [Jolt] and [Json-Sql] SmartModules used by this example connector -Example output - +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import SmDownloadJoltOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt-output.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmDownloadJsonSqlOutput from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql-output.bash'; -```bash -$ fluvio hub sm download infinyon/json-sql@x.y.z -downloading infinyon/json-sql@x.y.z to infinyon-json-sql-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +$ {SmDownloadJsonSql} $ {SmDownloadJolt} +Expect the following output: -```bash -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` + + {SmDownloadJsonSqlOutput} + {SmDownloadJoltOutput} + ##### Connector with JSON to JSON transformation config diff --git a/versioned_docs/version-0.13.0/fluvio/cli/fluvio/hub/list.mdx b/versioned_docs/version-0.13.0/fluvio/cli/fluvio/hub/list.mdx index 38c7b64f..cb0d9667 100644 --- a/versioned_docs/version-0.13.0/fluvio/cli/fluvio/hub/list.mdx +++ b/versioned_docs/version-0.13.0/fluvio/cli/fluvio/hub/list.mdx @@ -18,7 +18,7 @@ Listing Hub SmartModule from the CLI is straight forward: $ fluvio hub list SMARTMODULE Visibility - infinyon/jolt@0.1.0 public + infinyon/jolt@0.4.1 public infinyon/json-sql@0.1.0 public infinyon/regex-filter@0.1.0 public ... diff --git a/versioned_docs/version-0.13.0/fluvio/concepts/transformations.mdx b/versioned_docs/version-0.13.0/fluvio/concepts/transformations.mdx index db57df5e..85e6a92b 100644 --- a/versioned_docs/version-0.13.0/fluvio/concepts/transformations.mdx +++ b/versioned_docs/version-0.13.0/fluvio/concepts/transformations.mdx @@ -22,7 +22,7 @@ Example: ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: default @@ -30,14 +30,14 @@ transforms: source: "http" ``` -we have one transformation, which is a SmartModule named [`infinyon/jolt@0.3.0`]. +we have one transformation, which is a SmartModule named [`infinyon/jolt@0.4.1`]. The name must match a SmartModule previously downloaded to the Cluster: ```bash fluvio sm list SMARTMODULE SIZE - infinyon/jolt@0.3.0 608.4 KB + infinyon/jolt@0.4.1 608.4 KB ``` @@ -93,7 +93,7 @@ Multiple SmartModules can be activated in series. The output of the previous Sma ```yaml transforms: - - uses: infinyon/jolt@0.3.0 + - uses: infinyon/jolt@0.4.1 with: spec: - operation: shift @@ -108,4 +108,4 @@ transforms: In this example, the `jolt` transformation is performed first, then its output is the input the `regex-filter`. [SmartModules]: smartmodules/overview.mdx -[`infinyon/jolt@0.3.0`]: hub/smartmodules/jolt.mdx \ No newline at end of file +[`infinyon/jolt@0.4.1`]: hub/smartmodules/jolt.mdx \ No newline at end of file diff --git a/versioned_docs/version-0.13.0/hub/smartmodules/jolt.mdx b/versioned_docs/version-0.13.0/hub/smartmodules/jolt.mdx index abb1b1b8..ff7dd0eb 100644 --- a/versioned_docs/version-0.13.0/hub/smartmodules/jolt.mdx +++ b/versioned_docs/version-0.13.0/hub/smartmodules/jolt.mdx @@ -52,22 +52,23 @@ The Jolt SmartModule applies the operations in sequence: `remove` followed by `s To demonstrate how **Jolt** SmartModule works we will use [SMDK] tool. :::info -All smartmodule versions are marked with `x.y.z`. To find the latest version, run: +All smartmodule versions are marked with `0.4.1`. To find the latest version, run: * `fluvio hub smartmodule list` ::: First, we need to download it to our cluster: -```bash copy="fl" -$ fluvio hub smartmodule download infinyon/jolt@x.y.z -``` +import CodeBlock from '@theme/CodeBlock'; +import SmDownloadJolt from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; + +$ {SmDownloadJolt} Second, we create a file `transform.yaml` with transformation specification defined above: ```yaml # transform.yaml transforms: - - uses: infinyon/jolt@x.y.z + - uses: infinyon/jolt@0.4.1 spec: - operation: remove spec: @@ -113,4 +114,4 @@ For additional examples checkout the tutorials: [Fluvio Jolt]: https://github.com/infinyon/fluvio-jolt [map-type]:smartmodules/features/operators/map.mdx -[SMDK]: smartmodules/smdk.mdx \ No newline at end of file +[SMDK]: smartmodules/smdk.mdx diff --git a/versioned_docs/version-0.13.0/hub/use-the-hub.mdx b/versioned_docs/version-0.13.0/hub/use-the-hub.mdx index 0b8e1bbd..dacfe245 100644 --- a/versioned_docs/version-0.13.0/hub/use-the-hub.mdx +++ b/versioned_docs/version-0.13.0/hub/use-the-hub.mdx @@ -3,68 +3,91 @@ sidebar_position: 2 title: "Use the Hub" description: "The Hub is a central repository for Connectors, SmartModules, Dataflows, and more" --- +import CodeBlock from '@theme/CodeBlock'; +import HubConnList from '!!raw-loader!../_embeds/cmds/hub-conn-list.bash'; +import HubSmList from '!!raw-loader!../_embeds/cmds/hub-sm-list.bash'; +import HubSmDownloadJolt from '!!raw-loader!../_embeds/cmds/hub-sm-download-jolt.bash'; +import CdkDownloadHttpSink from '!!raw-loader!../_embeds/cmds/cdk-hub-download-http-sink.bash'; +import CdkDeployHttpSinkGen from '!!raw-loader!../_embeds/cmds/cdk-deploy-start-http-sink-generic.bash'; # How to use the Hub -The InfinyOn Hub serves as a centralized repository for InfinyOn tConnectors, SmartModules, and Dataflows. It facilitates the discovery and deployment of extensions to enhance the data streaming capabilities of Fluvio. The hub provides a platform for developers and data engineers to easily access pre-built solutions or share their own extensions. +The InfinyOn Hub serves as a centralized repository for InfinyOn [Connectors](#connectors), [SmartModules](#smartmodules), and Dataflows. It facilitates the discovery and deployment of extensions to enhance Fluvio's data streaming capabilities. The hub provides developers and data engineers a platform to access pre-built solutions or share their extensions easily. -## List +## Connectors -List the SmartModules available for download. +List the Connectors available for download. The official connectors (prepended with `infinyon/`) can run on InfinyOn Cloud and in your local environment. -```shell copy="fl" -$ fluvio hub smartmodule list - SMARTMODULE Visibility - infinyon-labs/array-map-json@x.y.z public - infinyon-labs/dedup-filter@x.y.z public - infinyon-labs/json-formatter@x.y.z public - infinyon-labs/key-gen-json@x.y.z public - infinyon-labs/regex-map-json@x.y.z public - infinyon-labs/regex-map@x.y.z public - infinyon-labs/stars-forks-changes@x.y.z public - infinyon/jolt@x.y.z public - infinyon/json-sql@x.y.z public - infinyon/regex-filter@x.y.z public -``` +### List Connectors -List the Connectors available for download. +Use `fluvio hub connector list` or `cdk hub list` to list the available connectors. -```shell copy="fl" -$ fluvio hub connector list - CONNECTOR Visibility - infinyon-labs/graphite-sink@x.y.z public - infinyon/duckdb-sink@x.y.z public - infinyon/http-sink@x.y.z public - infinyon/http-source@x.y.z public - infinyon/ic-webhook-source@x.y.z public - infinyon/kafka-sink@x.y.z public - infinyon/kafka-source@x.y.z public - infinyon/mqtt-source@x.y.z public - infinyon/sql-sink@x.y.z public -``` +$ {HubConnList} +Check out [Connectors for Developers] to learn how to build and publish your own connectors for the Hub. -## Download +### Use Connectors -Downloading a SmartModule to a target cluster defined by your [`profile`]. SmartModules need to be downloaded before they are used in transformations. +If you use a connector in InfinyOn Cloud, the Cloud infrastructure will automatically download and run the connector package. If you use connectors in your local environment, you must use the `cdk` command to download and run the connector package. -```shell copy="fl" -$ fluvio hub sm download infinyon/jolt@x.y.z -downloading infinyon/jolt@x.y.z to infinyon-jolt-x.y.z.ipkg -... downloading complete -... checking package -trying connection to fluvio router.infinyon.cloud:9003 -... cluster smartmodule install complete -``` +#### Download and Run Locally + +Connectors are independent binaries that run outside of Fluvio. Use the `cdk` command to download and run connectors locally: + +$ {CdkDownloadHttpSink} + +The same connector configuration file can run the connector locally or in InfinyOn Cloud. + +$ {CdkDeployHttpSinkGen} + +To stop a connector, use the `cdk deploy shutdown` command. + + +#### Deploy on InfinyOn Cloud -Running `fluvio hub conn download` will download the package containing source code of the Connector. +Connectors are automatically downloaded to InfinyOn Cloud. The management of InfinyOn Cloud is performed using the `fluvio cloud` commands. To run a connector in InfinyOn Cloud, use the command `fluvio cloud connector create`. ```shell copy="fl" -$ fluvio hub conn download infinyon/http-source@x.y.z -downloading infinyon/http-source@x.y.z to infinyon-http-source-x.y.z.ipkg -... downloading complete +$ fluvio cloud connector create --config ``` +**Note** List your profiles (`fluvio profile list`) to ensure your CLI points to the InfinyOn Cloud cluster. - [`profile`]: fluvio/cli/fluvio/profile.mdx \ No newline at end of file + +## SmartModules + +SmartModules are custom-defined programs that connectors use to transform data records during processing. Similar to connectors, SmartModules can be published to the Hub and made available for download. However, unlike connectors, SmartModules are downloaded to the cluster and are not executed outside of Fluvio. + +### List SmartModules + +List the SmartModules available for download. The official SmartModules (prepended with `infinyon/`). + +$ {HubSmList} + + +### Download and Run SmartModules + +Use `fluvio hub smartmodule download` to download specific SmartModules to a local or InfinyOn Cloud cluster. Use `fluvio profile list` to identify the target cluster. + +$ {HubSmDownloadJolt} + +SmartModules downloaded to a cluster can be used by the producer CLI or consumer CLI to transform records. They can also be accessed within the Connectors via the configuration file. + +To see the SmartModules currently installed on the cluster, use the following command: + +```shell copy="fl" +$ fluvio smartmodule list + ``` + +### References +- [Connector Developer Kit (cdk)] +- [Connector Configuration File] +- [SmartModule Developer Kit (smdk)] + +[cdk deploy shutdown]: /connectors/developers/overview.mdx#shutdown-a-connector +[`profile`]: fluvio/cli/fluvio/profile.mdx +[Connectors for Developers]: /connectors/developers/overview.mdx +[Connector Configuration File]: connectors/configuration.mdx +[Connector Developer Kit (cdk)]: /connectors/developers/overview.mdx +[SmartModule Developer Kit (smdk)]: smartmodules/developers/overview.mdx