From 98fae6da5d132c6bb7969cb5e1df4d080e90b9d1 Mon Sep 17 00:00:00 2001 From: "A.J. Hunyady" Date: Tue, 20 Aug 2024 18:43:44 -0700 Subject: [PATCH] Fix embeds with relative path to honor versioning. (#212) --- ...loy-start-http-source-cat-facts-xform.bash | 0 ...dk-deploy-start-http-source-cat-facts.bash | 0 .../cdk-deploy-start-sql-sink-cat-facts.bash | 0 .../cmds/cdk-hub-download-http-source.bash | 0 .../cmds/cdk-hub-download-sql-sink.bash | 0 .../_embeds}/cmds/hub-sm-download-jolt.bash | 0 .../cmds/hub-sm-download-json-sql.bash | 0 .../_embeds}/cmds/sm-list-jolt.bash | 0 .../_embeds}/cmds/sm-list-json-sql-jolt.bash | 0 .../connectors}/http-cat-fact-basic.yaml | 3 +- .../connectors/http-cat-fact-transform.yaml | 1 - .../_embeds}/connectors/sql-cat-fact.yaml | 0 {embeds => docs/_embeds}/versions.tsx | 0 docs/connectors/how-to/run-local.mdx | 2 +- .../tutorials/connector-transformations.mdx | 20 ++--- docs/fluvio/tutorials/http-source.mdx | 12 ++- docs/fluvio/tutorials/sql-sink.mdx | 26 ++---- ...loy-start-http-source-cat-facts-xform.bash | 1 + ...dk-deploy-start-http-source-cat-facts.bash | 1 + .../cdk-deploy-start-sql-sink-cat-facts.bash | 1 + .../cmds/cdk-hub-download-http-source.bash | 1 + .../cmds/cdk-hub-download-sql-sink.bash | 1 + .../_embeds/cmds/hub-sm-download-jolt.bash | 1 + .../cmds/hub-sm-download-json-sql.bash | 1 + .../_embeds/cmds/sm-list-jolt.bash | 3 + .../_embeds/cmds/sm-list-json-sql-jolt.bash | 4 + .../connectors/http-cat-fact-basic.yaml | 0 .../connectors}/http-cat-fact-transform.yaml | 3 +- .../connectors}/sql-cat-fact.yaml | 0 .../version-0.11.11/_embeds/versions.tsx | 5 ++ .../connectors/how-to/run-local.mdx | 3 +- ...tion.mdx => connector-transformations.mdx} | 73 +++++++++-------- ...{source-http-basic.mdx => http-source.mdx} | 82 +++++++++++-------- .../fluvio/tutorials/mirroring-iot-local.mdx | 6 +- .../tutorials/mirroring-two-clusters.mdx | 8 +- .../{output-sql.mdx => sql-sink.mdx} | 61 +++++++------- 36 files changed, 165 insertions(+), 154 deletions(-) rename {embeds => docs/_embeds}/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash (100%) rename {embeds => docs/_embeds}/cmds/cdk-deploy-start-http-source-cat-facts.bash (100%) rename {embeds => docs/_embeds}/cmds/cdk-deploy-start-sql-sink-cat-facts.bash (100%) rename {embeds => docs/_embeds}/cmds/cdk-hub-download-http-source.bash (100%) rename {embeds => docs/_embeds}/cmds/cdk-hub-download-sql-sink.bash (100%) rename {embeds => docs/_embeds}/cmds/hub-sm-download-jolt.bash (100%) rename {embeds => docs/_embeds}/cmds/hub-sm-download-json-sql.bash (100%) rename {embeds => docs/_embeds}/cmds/sm-list-jolt.bash (100%) rename {embeds => docs/_embeds}/cmds/sm-list-json-sql-jolt.bash (100%) rename {versioned_docs/version-0.11.11/fluvio/tutorials/config => docs/_embeds/connectors}/http-cat-fact-basic.yaml (91%) rename {embeds => docs/_embeds}/connectors/http-cat-fact-transform.yaml (93%) rename {embeds => docs/_embeds}/connectors/sql-cat-fact.yaml (100%) rename {embeds => docs/_embeds}/versions.tsx (100%) create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-sql-sink.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/sm-list-jolt.bash create mode 100644 versioned_docs/version-0.11.11/_embeds/cmds/sm-list-json-sql-jolt.bash rename {embeds => versioned_docs/version-0.11.11/_embeds}/connectors/http-cat-fact-basic.yaml (100%) rename versioned_docs/version-0.11.11/{fluvio/tutorials/config => _embeds/connectors}/http-cat-fact-transform.yaml (86%) rename versioned_docs/version-0.11.11/{fluvio/tutorials/config => _embeds/connectors}/sql-cat-fact.yaml (100%) create mode 100644 versioned_docs/version-0.11.11/_embeds/versions.tsx rename versioned_docs/version-0.11.11/fluvio/tutorials/{source-http-transformation.mdx => connector-transformations.mdx} (66%) rename versioned_docs/version-0.11.11/fluvio/tutorials/{source-http-basic.mdx => http-source.mdx} (70%) rename versioned_docs/version-0.11.11/fluvio/tutorials/{output-sql.mdx => sql-sink.mdx} (81%) diff --git a/embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash b/docs/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash similarity index 100% rename from embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash rename to docs/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash diff --git a/embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash b/docs/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash similarity index 100% rename from embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash rename to docs/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash diff --git a/embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash b/docs/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash similarity index 100% rename from embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash rename to docs/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash diff --git a/embeds/cmds/cdk-hub-download-http-source.bash b/docs/_embeds/cmds/cdk-hub-download-http-source.bash similarity index 100% rename from embeds/cmds/cdk-hub-download-http-source.bash rename to docs/_embeds/cmds/cdk-hub-download-http-source.bash diff --git a/embeds/cmds/cdk-hub-download-sql-sink.bash b/docs/_embeds/cmds/cdk-hub-download-sql-sink.bash similarity index 100% rename from embeds/cmds/cdk-hub-download-sql-sink.bash rename to docs/_embeds/cmds/cdk-hub-download-sql-sink.bash diff --git a/embeds/cmds/hub-sm-download-jolt.bash b/docs/_embeds/cmds/hub-sm-download-jolt.bash similarity index 100% rename from embeds/cmds/hub-sm-download-jolt.bash rename to docs/_embeds/cmds/hub-sm-download-jolt.bash diff --git a/embeds/cmds/hub-sm-download-json-sql.bash b/docs/_embeds/cmds/hub-sm-download-json-sql.bash similarity index 100% rename from embeds/cmds/hub-sm-download-json-sql.bash rename to docs/_embeds/cmds/hub-sm-download-json-sql.bash diff --git a/embeds/cmds/sm-list-jolt.bash b/docs/_embeds/cmds/sm-list-jolt.bash similarity index 100% rename from embeds/cmds/sm-list-jolt.bash rename to docs/_embeds/cmds/sm-list-jolt.bash diff --git a/embeds/cmds/sm-list-json-sql-jolt.bash b/docs/_embeds/cmds/sm-list-json-sql-jolt.bash similarity index 100% rename from embeds/cmds/sm-list-json-sql-jolt.bash rename to docs/_embeds/cmds/sm-list-json-sql-jolt.bash diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-basic.yaml b/docs/_embeds/connectors/http-cat-fact-basic.yaml similarity index 91% rename from versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-basic.yaml rename to docs/_embeds/connectors/http-cat-fact-basic.yaml index 6a4c74ea..40d93766 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-basic.yaml +++ b/docs/_embeds/connectors/http-cat-fact-basic.yaml @@ -1,4 +1,3 @@ - apiVersion: 0.1.0 meta: version: 0.3.8 @@ -9,4 +8,4 @@ meta: http: endpoint: "https://catfact.ninja/fact" - interval: 10s + interval: 10s \ No newline at end of file diff --git a/embeds/connectors/http-cat-fact-transform.yaml b/docs/_embeds/connectors/http-cat-fact-transform.yaml similarity index 93% rename from embeds/connectors/http-cat-fact-transform.yaml rename to docs/_embeds/connectors/http-cat-fact-transform.yaml index b8df8bbc..9aff771d 100644 --- a/embeds/connectors/http-cat-fact-transform.yaml +++ b/docs/_embeds/connectors/http-cat-fact-transform.yaml @@ -4,7 +4,6 @@ meta: name: cat-facts-transformed type: http-source topic: cat-facts-data-transform - create-topic: true http: endpoint: https://catfact.ninja/fact diff --git a/embeds/connectors/sql-cat-fact.yaml b/docs/_embeds/connectors/sql-cat-fact.yaml similarity index 100% rename from embeds/connectors/sql-cat-fact.yaml rename to docs/_embeds/connectors/sql-cat-fact.yaml diff --git a/embeds/versions.tsx b/docs/_embeds/versions.tsx similarity index 100% rename from embeds/versions.tsx rename to docs/_embeds/versions.tsx diff --git a/docs/connectors/how-to/run-local.mdx b/docs/connectors/how-to/run-local.mdx index db1ca3f4..42e86953 100644 --- a/docs/connectors/how-to/run-local.mdx +++ b/docs/connectors/how-to/run-local.mdx @@ -3,7 +3,7 @@ sidebar_position: 10 title: Run Local description: "Run a Connector on your local machine" --- -import Versions from '@site/embeds/versions'; +import Versions from '../../_embeds/versions'; # Run Connectors on Your Local Machine diff --git a/docs/fluvio/tutorials/connector-transformations.mdx b/docs/fluvio/tutorials/connector-transformations.mdx index 18bed41e..efd8c994 100644 --- a/docs/fluvio/tutorials/connector-transformations.mdx +++ b/docs/fluvio/tutorials/connector-transformations.mdx @@ -4,6 +4,11 @@ title: "Connector Transformations" description: "Part 2 of 3: HTTP to SQL Sink tutorial series." --- import CodeBlock from '@theme/CodeBlock'; +import HubSmDownload from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import HubSmList from '!!raw-loader!../../_embeds/cmds/sm-list-jolt.bash'; +import CatFactBasic from '!!raw-loader!../../_embeds/connectors/http-cat-fact-transform.yaml'; +import CdkDeployHttpSrc from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash'; + # Connector Transformations @@ -38,17 +43,10 @@ The SmartModule we will use in this tutorial implements domain specific language To use jolt SmartModule, we need to download it to the cluster. - -import HubSmDownload from '!!raw-loader!@site/embeds/cmds/hub-sm-download-jolt.bash'; - $ {HubSmDownload} - - Once you have the SmartModule downloaded, you can list them: -import HubSmList from '!!raw-loader!@site/embeds/cmds/sm-list-jolt.bash'; - $ {HubSmList} @@ -60,22 +58,14 @@ Before we start modified connector, please terminate existing running connector. Coyp and paste following config and save it as `http-cat-facts-transform.yaml`. This configuration use the same endpoint as the previous tutorial except different topic name. - -import CatFactBasic from '!!raw-loader!@site/embeds/connectors/http-cat-fact-transform.yaml'; - {CatFactBasic} - Noticed that we just added `transforms` section which added JOLT specification to insert a new field `source` with value `http` to every record. Once you have the config file, you can create the connector using the `cdk deploy start` command as before. - -import CdkDeployHttpSrc from '!!raw-loader!@site/embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash'; - $ {CdkDeployHttpSrc} - You can use `cdk deploy list` to view the status of the connector. ```bash copy="fl" diff --git a/docs/fluvio/tutorials/http-source.mdx b/docs/fluvio/tutorials/http-source.mdx index 5d9e4675..76d2a2c5 100644 --- a/docs/fluvio/tutorials/http-source.mdx +++ b/docs/fluvio/tutorials/http-source.mdx @@ -4,7 +4,11 @@ title: "HTTP Source -> Topic" description: "Part 1 of 3: HTTP to SQL Sink tutorial series." --- import CodeBlock from '@theme/CodeBlock'; -import Versions from '@site/embeds/versions'; +import Versions from '../../_embeds/versions'; +import CatFactBasic from '!!raw-loader!../../_embeds/connectors/http-cat-fact-basic.yaml'; +import CdkHubDownload from '!!raw-loader!../../_embeds/cmds/cdk-hub-download-http-source.bash'; +import CdkDeployHttpSrc from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash'; + # HTTP Source -> Topic @@ -54,8 +58,6 @@ The connector you downloaded contains binary executable. The connector is packa In this tutorial, we will be using the `HTTP Source Connector` from the Hub. -import CdkHubDownload from '!!raw-loader!@site/embeds/cmds/cdk-hub-download-http-source.bash'; - $ {CdkHubDownload} @@ -69,8 +71,6 @@ After downloading, you need to create a configuration file to provision the conn Coyp and paste following config and save it as `http-cat-facts.yaml`. -import CatFactBasic from '!!raw-loader!@site/embeds/connectors/http-cat-fact-basic.yaml'; - {CatFactBasic} @@ -86,8 +86,6 @@ The `interval` is set to `10s` which means the connector will poll the endpoint Once you have the config file, you can create the connector using the `cdk deploy start` command. -import CdkDeployHttpSrc from '!!raw-loader!@site/embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash'; - $ {CdkDeployHttpSrc} diff --git a/docs/fluvio/tutorials/sql-sink.mdx b/docs/fluvio/tutorials/sql-sink.mdx index ef870d6d..18a1afff 100644 --- a/docs/fluvio/tutorials/sql-sink.mdx +++ b/docs/fluvio/tutorials/sql-sink.mdx @@ -4,6 +4,12 @@ title: "Topic -> SQL Sink" description: "Part 1 of 3: HTTP to SQL Sink tutorial series." --- import CodeBlock from '@theme/CodeBlock'; +import HubDownloadSqlSink from '!!raw-loader!../../_embeds/cmds/cdk-hub-download-sql-sink.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmListJsonSqlJolt from '!!raw-loader!../../_embeds/cmds/sm-list-json-sql-jolt.bash'; +import CatSQL from '!!raw-loader!../../_embeds/connectors/sql-cat-fact.yaml'; +import CdkDeploySqlSink from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash'; + # Topic -> SQL Sink @@ -35,38 +41,22 @@ We will be using topics from first tutorial [Streaming from HTTP Source] which s As in previous tutorials, we will use `cdk` to manage the connectors. Run following command to download the connector from the Hub. - -import HubDownloadSqlSink from '!!raw-loader!@site/embeds/cmds/cdk-hub-download-sql-sink.bash'; - $ {HubDownloadSqlSink} - Then download SQL SmartModule from the Hub. - -import SmDownloadJsonSql from '!!raw-loader!@site/embeds/cmds/hub-sm-download-json-sql.bash'; - $ {SmDownloadJsonSql} - Then you should see two smartmodules downloaded assuming you have already downloaded the `jolt` SmartModule from previous tutorial. - -import SmListJsonSqlJolt from '!!raw-loader!@site/embeds/cmds/sm-list-json-sql-jolt.bash'; - $ {SmListJsonSqlJolt} - ## Sink Connector configuration Coyp and paste following config and save it as `sql-cat-fact.yaml`. - -import CatSQL from '!!raw-loader!@site/embeds/connectors/sql-cat-fact.yaml'; - {CatSQL} - This configuration will read data from `cat-facts` topic and insert into `animalfacts` table in the database. The `json-sql` SmartModule will transform the JSON data into SQL insert statement. Please change line containing `url` to your database connection string. @@ -111,12 +101,8 @@ You can confirm table is created: Once you have the config file, you can create the connector using the `cdk deploy start` command. - -import CdkDeploySqlSink from '!!raw-loader!@site/embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash'; - $ {CdkDeploySqlSink} - You can use `cdk deploy list` to view the status of the connector. ```bash copy="fl" diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash new file mode 100644 index 00000000..c8cec794 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-source-0.3.8.ipkg --config ./http-cat-facts-transform.yaml \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash new file mode 100644 index 00000000..d0ae93f0 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-http-source-0.3.8.ipkg --config ./http-cat-facts.yaml \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash new file mode 100644 index 00000000..2e68256d --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash @@ -0,0 +1 @@ +cdk deploy start --ipkg infinyon-sql-sink-0.4.3.ipkg --config ./sql-cat-fact.yaml \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash new file mode 100644 index 00000000..76576b6a --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-http-source.bash @@ -0,0 +1 @@ +cdk hub download infinyon/http-source@0.3.8 \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-sql-sink.bash b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-sql-sink.bash new file mode 100644 index 00000000..6076dcd3 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/cdk-hub-download-sql-sink.bash @@ -0,0 +1 @@ +cdk hub download infinyon/sql-sink@0.4.3 \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash new file mode 100644 index 00000000..399fa672 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-jolt.bash @@ -0,0 +1 @@ +fluvio hub sm download infinyon/jolt@0.4.1 \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql.bash b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql.bash new file mode 100644 index 00000000..1c786e2f --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/hub-sm-download-json-sql.bash @@ -0,0 +1 @@ +fluvio hub sm download infinyon/json-sql@0.2.1 \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-jolt.bash b/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-jolt.bash new file mode 100644 index 00000000..f24a6932 --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-jolt.bash @@ -0,0 +1,3 @@ +fluvio sm list + SMARTMODULE SIZE + infinyon/jolt@0.4.1 589.3 KB \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-json-sql-jolt.bash b/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-json-sql-jolt.bash new file mode 100644 index 00000000..2a1b365e --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/cmds/sm-list-json-sql-jolt.bash @@ -0,0 +1,4 @@ +fluvio sm list + SMARTMODULE SIZE + infinyon/json-sql@0.2.1 559.6 KB + infinyon/jolt@0.4.1 589.3 KB \ No newline at end of file diff --git a/embeds/connectors/http-cat-fact-basic.yaml b/versioned_docs/version-0.11.11/_embeds/connectors/http-cat-fact-basic.yaml similarity index 100% rename from embeds/connectors/http-cat-fact-basic.yaml rename to versioned_docs/version-0.11.11/_embeds/connectors/http-cat-fact-basic.yaml diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-transform.yaml b/versioned_docs/version-0.11.11/_embeds/connectors/http-cat-fact-transform.yaml similarity index 86% rename from versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-transform.yaml rename to versioned_docs/version-0.11.11/_embeds/connectors/http-cat-fact-transform.yaml index a9c42aa2..9aff771d 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/config/http-cat-fact-transform.yaml +++ b/versioned_docs/version-0.11.11/_embeds/connectors/http-cat-fact-transform.yaml @@ -4,7 +4,6 @@ meta: name: cat-facts-transformed type: http-source topic: cat-facts-data-transform - create-topic: true http: endpoint: https://catfact.ninja/fact @@ -16,4 +15,4 @@ transforms: spec: - operation: default spec: - source: "http" + source: "http" \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/config/sql-cat-fact.yaml b/versioned_docs/version-0.11.11/_embeds/connectors/sql-cat-fact.yaml similarity index 100% rename from versioned_docs/version-0.11.11/fluvio/tutorials/config/sql-cat-fact.yaml rename to versioned_docs/version-0.11.11/_embeds/connectors/sql-cat-fact.yaml diff --git a/versioned_docs/version-0.11.11/_embeds/versions.tsx b/versioned_docs/version-0.11.11/_embeds/versions.tsx new file mode 100644 index 00000000..0ecc077e --- /dev/null +++ b/versioned_docs/version-0.11.11/_embeds/versions.tsx @@ -0,0 +1,5 @@ +const versions = { + infinyon_http_source: 'infinyon-http-source@0.3.8', +} + +export default versions; \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/connectors/how-to/run-local.mdx b/versioned_docs/version-0.11.11/connectors/how-to/run-local.mdx index 1996df3e..5f10fb8c 100644 --- a/versioned_docs/version-0.11.11/connectors/how-to/run-local.mdx +++ b/versioned_docs/version-0.11.11/connectors/how-to/run-local.mdx @@ -3,8 +3,7 @@ sidebar_position: 10 title: Run Local description: "Run a Connector on your local machine" --- - -import Versions from '@site/embeds/versions'; +import Versions from '../../_embeds/versions'; # Run Connectors on Your Local Machine diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/source-http-transformation.mdx b/versioned_docs/version-0.11.11/fluvio/tutorials/connector-transformations.mdx similarity index 66% rename from versioned_docs/version-0.11.11/fluvio/tutorials/source-http-transformation.mdx rename to versioned_docs/version-0.11.11/fluvio/tutorials/connector-transformations.mdx index 222dc3b5..efd8c994 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/source-http-transformation.mdx +++ b/versioned_docs/version-0.11.11/fluvio/tutorials/connector-transformations.mdx @@ -1,30 +1,41 @@ --- -sidebar_position: 2 -title: "Customizing Connector with Transformation" -description: "Part 2 of HTTP to SQL Sink tutorial series." +sidebar_position: 11 +title: "Connector Transformations" +description: "Part 2 of 3: HTTP to SQL Sink tutorial series." --- +import CodeBlock from '@theme/CodeBlock'; +import HubSmDownload from '!!raw-loader!../../_embeds/cmds/hub-sm-download-jolt.bash'; +import HubSmList from '!!raw-loader!../../_embeds/cmds/sm-list-jolt.bash'; +import CatFactBasic from '!!raw-loader!../../_embeds/connectors/http-cat-fact-transform.yaml'; +import CdkDeployHttpSrc from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-http-source-cat-facts-xform.bash'; + -# Prerequisites +# Connector Transformations -This guide uses `local` Fluvio cluster. If you need to install it, please follow the instructions [here][installation]! +This tutorial is part of the **HTTP website to SQL database** series: +* [Part 1: HTTP Source -> Topic] +* **Part 2: Connector Transformations** +* [Part 3: Topic -> SQL Sink] -# Introduction +## Introduction -Building on the [Streaming from HTTP Source] tutorial, we will customize the Source HTTP Connector with transformation that will let you modify the records before they are sent to the topic. +Building on the [HTTP Source -> Topic] tutorial, we will customize the Source HTTP Connector with transformations to modify the records before sending them to the topic. -Why would you want to do this? +#### Why would you want to do this? Often, you are only interested in a subset of the data, or add/remove fields, or even change the shape of the record. This is where transformations come in. -In this tutorial, we will show you how to customize the HTTP Source Connector to add a field to every record before it is sent to the topic. +#### Prerequisites + +You will need a `local` fluvio cluster and `cdk` to follow along. Follow the [installation instructions][installation] if you don't have them already. -# SmartModule +## SmartModules -SmartModule is a reusable piece of code that can be attached to a connector to perform a specific task. In this case, we will use a SmartModule to transform the incoming JSON records before they are sent to the topic. +SmartModules are reusable pieces of code that can be attached to a connector to perform specific tasks. In this case, we will use a SmartModule to transform the incoming JSON records before we send it to the topic. -SmartModule is executed using WebAssembly runtime, which means it is very small and secure. You can write your own SmartModules in Rust, or use the prepackaged ones from the [SmartModule Hub]. +SmartModules are executed using the WebAssembly runtime, which means they are small and secure. You can write your own SmartModule in Rust, or use the prepackaged ones from the [SmartModule Hub]. Unlike connectors, SmartModules in the Hub are packaged WASM files. @@ -32,20 +43,14 @@ The SmartModule we will use in this tutorial implements domain specific language To use jolt SmartModule, we need to download it to the cluster. -```bash -$ fluvio hub sm download infinyon/jolt@0.4.1 -``` +$ {HubSmDownload} Once you have the SmartModule downloaded, you can list them: -```bash -$ fluvio hub sm list - SMARTMODULE SIZE - infinyon/jolt@0.4.1 589.3 KB -``` +$ {HubSmList} -# Customizing HTTP Connector with JOLT SmartModule +## Customizing HTTP Connector with JOLT SmartModule In order to customize the HTTP Source Connector, we need to create a configuration file that includes the transformation. @@ -53,33 +58,27 @@ Before we start modified connector, please terminate existing running connector. Coyp and paste following config and save it as `http-cat-facts-transform.yaml`. This configuration use the same endpoint as the previous tutorial except different topic name. -import CodeBlock from '@theme/CodeBlock'; -import CatFactBasic from '!!raw-loader!./config/http-cat-fact-transform.yaml'; - {CatFactBasic} Noticed that we just added `transforms` section which added JOLT specification to insert a new field `source` with value `http` to every record. Once you have the config file, you can create the connector using the `cdk deploy start` command as before. -```bash -$ cdk deploy start --ipkg infinyon-http-source-0.3.8.ipkg --config ./http-cat-facts-transform.yaml -``` - +$ {CdkDeployHttpSrc} You can use `cdk deploy list` to view the status of the connector. -```bash +```bash copy="fl" $ cdk deploy list NAME STATUS cat-facts Running ``` -# Checking the data +## Check the data Similar to the previous tutorial, we can use `fluvio consume` to view the incoming data in the topic `cat-facts-transformed` which was created by the connector. -```bash +```bash copy="fl" $ fluvio consume cat-facts -T4 Consuming records starting 4 from the end of topic 'cat-facts' @@ -93,15 +92,16 @@ As you can see, the `source` field has been added to every record. This is the t You can terminate the consume command by pressing `Ctrl+C`. -### Cleaning up +## Clean up Shut down the connector and delete the topic. -```bash +```bash copy="full" $ cdk deploy shutdown --name cat-facts-transformed $ fluvio topic delete cat-facts-transform ``` + ## Conclusion and Next Step In this tutorial, we showed you how to customize the HTTP Source Connector with transformation using SmartModule. @@ -111,7 +111,7 @@ SmartModule is a powerful tool that can be used to perform complex transformatio In the next tutorial, we will show you how to use the sink connector to stream data from an topic to a SQL database. -## Reference +## References * [Fluvio CLI Produce] * [Fluvio CLI Consume] @@ -121,7 +121,10 @@ In the next tutorial, we will show you how to use the sink connector to stream d * [SmartModule Rust API] * [Transformations] -[Streaming from HTTP Source]: source-http-basic.mdx + +[Part 1: HTTP Source -> Topic]: http-source.mdx +[HTTP Source -> Topic]: http-source.mdx +[Part 3: Topic -> SQL Sink]: sql-sink.mdx [Connector Overview]: connectors/overview.mdx [Fluvio CLI Produce]: fluvio/cli/fluvio/produce.mdx [Fluvio CLI Consume]: fluvio/cli/fluvio/consume.mdx diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/source-http-basic.mdx b/versioned_docs/version-0.11.11/fluvio/tutorials/http-source.mdx similarity index 70% rename from versioned_docs/version-0.11.11/fluvio/tutorials/source-http-basic.mdx rename to versioned_docs/version-0.11.11/fluvio/tutorials/http-source.mdx index 9b473727..76d2a2c5 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/source-http-basic.mdx +++ b/versioned_docs/version-0.11.11/fluvio/tutorials/http-source.mdx @@ -1,28 +1,39 @@ --- -sidebar_position: 1 -title: "Streaming from HTTP Source" -description: "Part 1 of HTTP to SQL Sink tutorial series." +sidebar_position: 10 +title: "HTTP Source -> Topic" +description: "Part 1 of 3: HTTP to SQL Sink tutorial series." --- +import CodeBlock from '@theme/CodeBlock'; +import Versions from '../../_embeds/versions'; +import CatFactBasic from '!!raw-loader!../../_embeds/connectors/http-cat-fact-basic.yaml'; +import CdkHubDownload from '!!raw-loader!../../_embeds/cmds/cdk-hub-download-http-source.bash'; +import CdkDeployHttpSrc from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-http-source-cat-facts.bash'; -# Prerequisites -This guide uses `local` Fluvio cluster. If you need to install it, please follow the instructions [here][installation]! +# HTTP Source -> Topic +This tutorial is part of the **HTTP website to SQL database** series: +* **Part 1: HTTP Source -> Topic** +* [Part 2: Connector Transformations] +* [Part 3: Topic -> SQL Sink] # Introduction -This tutorial will guide you through creating a simple data pipeline that streams data from a website but can be applied to any HTTP source. +This tutorial guides you through creating a simple data pipeline that streams data from a website through the [HTTP source connector]. + +#### Tutorial will cover -Tutorial will cover: -- Connector - How to download and start an Inbound HTTP Connector -- Reading data from the topic -- Stopping the connector +- Read data from the topic +- Stop the connector +#### Prerequisites +You will need a `local` fluvio cluster and `cdk` to follow along. Follow the [installation instructions][installation] if you don't have them already. -# Connector Overview + +## Connector Overview The Fluvio connector lets fluvio to interact with external systems. There are two types of connectors: `Source` and `Sink`. @@ -47,23 +58,22 @@ The connector you downloaded contains binary executable. The connector is packa In this tutorial, we will be using the `HTTP Source Connector` from the Hub. -```bash -$ cdk hub download infinyon/http-source@0.3.8 -``` +$ {CdkHubDownload} -This will download `infinyon-http-source-0.3.8.ipkg` file which contains the connector binary. -# Configuration for HTTP Source Connector +This command will download the {Versions.infinyon_http_source}.ipkg file containing the connector binary. -After downloading the connector, you need to create a configuration file for the connector. Following is the basic configuration file for the `HTTP Source Connector` that reads data from [castfact.ninja] website. + +## Configure for HTTP Source Connector + +After downloading, you need to create a configuration file to provision the connector. Following configuration file provisions the `HTTP Source Connector` to query periodically the [castfact.ninja] website. Coyp and paste following config and save it as `http-cat-facts.yaml`. -import CodeBlock from '@theme/CodeBlock'; -import CatFactBasic from '!!raw-loader!./config/http-cat-fact-basic.yaml'; {CatFactBasic} + Each connector has a `meta` section which is same across all connector. The `http` section is specific to the `HTTP Source Connector`. Noticed that in the `meta`, `create-type` is set to true which means the topic will be created if it does not exist. @@ -71,29 +81,29 @@ Noticed that in the `meta`, `create-type` is set to true which means the topic w The `interval` is set to `10s` which means the connector will poll the endpoint every 10 seconds. -# Starting the Connector +## Start the Connector Once you have the config file, you can create the connector using the `cdk deploy start` command. -```bash -$ cdk deploy start --ipkg infinyon-http-source-0.3.8.ipkg --config ./http-cat-facts.yaml -``` + +$ {CdkDeployHttpSrc} + You can use `cdk deploy list` to view the status of the connector. -```bash +```bash copy="fl" $ cdk deploy list NAME STATUS cat-facts Running ``` -# Checking the data +## Check the Data Use `fluvio consume` to view the incoming data in the topic `cat-facts` which was created by the connector. This command will consume the last 4 records from the topic in the stream mode. -```bash +```bash copy="fl" $ fluvio consume cat-facts -T4 Consuming records starting 4 from the end of topic 'cat-facts' {"fact":"A cat lover is called an Ailurophilia (Greek: cat+lover).","length":57} @@ -101,27 +111,28 @@ Consuming records starting 4 from the end of topic 'cat-facts' {"fact":"Fossil records from two million years ago show evidence of jaguars.","length":67} {"fact":"Relative to its body size, the clouded leopard has the biggest canines of all animals\u2019 canines. Its dagger-like teeth can be as long as 1.8 inches (4.5 cm).","length":156} ``` + The http connector will poll the endpoint every 10 seconds and write the data to the topic. As you can see, CLI output will refresh every 10 seconds with new data. You can terminate the consume command by pressing `Ctrl+C`. -### Cleaning Up +## Clean Up To shutdown traffic, you can shutdown local connector using the `cdk deploy shutdown` command. -```bash +```bash copy="fl" $ cdk deploy shutdown --name cat-facts ``` You can also delete the topic by using the `fluvio topic delete` command. -```bash +```bash copy="fl" $ fluvio topic delete cat-facts ``` This should result in no connectors running: -```bash +```bash copy="fl" $ cdk deploy list NAME STATUS ``` @@ -140,16 +151,21 @@ There are many other source connectors available in the Hub such as `mqtt` and ` In the next tutorial [Customizing HTTP Connector], we will introduce `SmartModule` which can be used to transform the data before writing it to the topic. -## Reference +## References * [Fluvio CLI Consume] * [Fluvio CLI Topic] * [Fluvio CLI Profile] -[Customizing HTTP Connector]: source-http-transformation.mdx + +[Part 2: Connector Transformations]: connector-transformations.mdx +[Part 3: Topic -> SQL Sink]: sql-sink.mdx +[Http source connector]: hub/connectors/inbound/http.mdx +[Customizing HTTP Connector]: connector-transformations.mdx [Connector Overview]: connectors/overview.mdx [Fluvio CLI Consume]: fluvio/cli/fluvio/consume.mdx [Fluvio CLI Topic]: fluvio/cli/fluvio/topic.mdx +[Fluvio CLI Profile]: fluvio/cli/fluvio/profile.mdx [Inbound HTTP Connector]: hub/connectors/inbound/http.mdx [castfact.ninja]: https://catfact.ninja -[installation]: fluvio/quickstart.mdx#install-fluvio +[installation]: fluvio/quickstart.mdx#install-fluvio \ No newline at end of file diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-iot-local.mdx b/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-iot-local.mdx index ade9b4a7..a3f106d7 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-iot-local.mdx +++ b/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-iot-local.mdx @@ -1,9 +1,11 @@ --- -sidebar_position: 10 -title: "Mirroring IoT Data from Raspberry Pi to Local Cluster" +sidebar_position: 70 +title: "Mirroring - Raspberry Pi to Cluster" description: "Reliable IoT monitoring from movable or static Edges sensors." --- +# Mirroring IoT Data from Raspberry Pi to Local Cluster + This advanced tutorial requires a Raspberry Pi and a local installation of your collector cluster running on Kubernetes. ## Raspberry Pi to Local Cluster diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-two-clusters.mdx b/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-two-clusters.mdx index 95ca28ac..eb6e14d2 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-two-clusters.mdx +++ b/versioned_docs/version-0.11.11/fluvio/tutorials/mirroring-two-clusters.mdx @@ -1,10 +1,12 @@ --- -sidebar_position: 7 -title: "Mirroring Two Local Clusters" +sidebar_position: 80 +title: "Mirroring - Cluster to Cluster" description: "Mirroring fluvio topics between two local clusters" --- -This tutorial is a demonstration of mirroring locally with two clusters on docker. +# Mirroring Two Local Clusters + +This tutorial is a demonstration of mirroring between two clusters running on Docker. ## Home and Remote diff --git a/versioned_docs/version-0.11.11/fluvio/tutorials/output-sql.mdx b/versioned_docs/version-0.11.11/fluvio/tutorials/sql-sink.mdx similarity index 81% rename from versioned_docs/version-0.11.11/fluvio/tutorials/output-sql.mdx rename to versioned_docs/version-0.11.11/fluvio/tutorials/sql-sink.mdx index dd54022b..18a1afff 100644 --- a/versioned_docs/version-0.11.11/fluvio/tutorials/output-sql.mdx +++ b/versioned_docs/version-0.11.11/fluvio/tutorials/sql-sink.mdx @@ -1,18 +1,29 @@ --- -sidebar_position: 3 -title: "Streaming data to SQL" -description: "Part 3 of HTTP to SQL Sink tutorial series." +sidebar_position: 20 +title: "Topic -> SQL Sink" +description: "Part 1 of 3: HTTP to SQL Sink tutorial series." --- +import CodeBlock from '@theme/CodeBlock'; +import HubDownloadSqlSink from '!!raw-loader!../../_embeds/cmds/cdk-hub-download-sql-sink.bash'; +import SmDownloadJsonSql from '!!raw-loader!../../_embeds/cmds/hub-sm-download-json-sql.bash'; +import SmListJsonSqlJolt from '!!raw-loader!../../_embeds/cmds/sm-list-json-sql-jolt.bash'; +import CatSQL from '!!raw-loader!../../_embeds/connectors/sql-cat-fact.yaml'; +import CdkDeploySqlSink from '!!raw-loader!../../_embeds/cmds/cdk-deploy-start-sql-sink-cat-facts.bash'; + +# Topic -> SQL Sink -# Prerequisites +This tutorial is part of the **HTTP website to SQL database** series: +* [Part 1: HTTP Source -> Topic] +* [Part 2: Connector Transformations] +* **Part 3: Topic -> SQL Sink** This guide uses `local` Fluvio cluster. If you need to install it, please follow the instructions [here][installation]! -We will be using `Postgres` database, You can download it and set up from [PostgreSQL] website for your OS. Alternatively use a cloud service like [ElephantSQL]. +We will be using `Postgres` database, You can download it and set up from [PostgreSQL] website for your OS. Alternatively use a cloud service like [ElephantSQL]. -# Introduction +## Introduction In previous tutorials, we have seen how to read data from external sources and write it to a Fluvio topic. In this tutorial, we will go through how to sink data from a Fluvio topic to external sink such as a database. @@ -30,32 +41,20 @@ We will be using topics from first tutorial [Streaming from HTTP Source] which s As in previous tutorials, we will use `cdk` to manage the connectors. Run following command to download the connector from the Hub. -```bash -$ cdk hub download infinyon/sql-sink@0.4.3 -``` +$ {HubDownloadSqlSink} Then download SQL SmartModule from the Hub. -```bash -$ fluvio hub sm download infinyon/json-sql@0.2.1 -``` +$ {SmDownloadJsonSql} Then you should see two smartmodules downloaded assuming you have already downloaded the `jolt` SmartModule from previous tutorial. -```bash -$ fluvio sm list - SMARTMODULE SIZE - infinyon/json-sql@0.2.1 559.6 KB - infinyon/jolt@0.4.1 589.3 KB -``` +$ {SmListJsonSqlJolt} -# Sink Connector configuration +## Sink Connector configuration Coyp and paste following config and save it as `sql-cat-fact.yaml`. -import CodeBlock from '@theme/CodeBlock'; -import CatSQL from '!!raw-loader!./config/sql-cat-fact.yaml'; - {CatSQL} This configuration will read data from `cat-facts` topic and insert into `animalfacts` table in the database. The `json-sql` SmartModule will transform the JSON data into SQL insert statement. @@ -81,7 +80,7 @@ For example, here is mapping for `length` column: This mapping will take `length` field from JSON and insert into `length` column in the table. If `length` field is not found, it will use default value of `0`. -# Setting up the Database +## Setup the Database In order to run the connector, you need to create a table in your database. Run following SQL command in postgres CLI: @@ -102,19 +101,17 @@ You can confirm table is created: Once you have the config file, you can create the connector using the `cdk deploy start` command. -```bash -$ cdk deploy start --ipkg infinyon-sql-sink-0.4.3.ipkg --config ./sql-cat-fact.yaml -``` +$ {CdkDeploySqlSink} You can use `cdk deploy list` to view the status of the connector. -```bash +```bash copy="fl" $ cdk deploy list NAME STATUS simple-cat-facts-sql Running ``` -# Generate data and checking the data +## Generate data and Check the result Fluvio topic allow you to decouple the data source from the data sink. This means both source and sink can be run independently without affecting each other. You can run the source connector to generate data but it is not required for this demo. @@ -124,7 +121,7 @@ By default, sink connector will consume the data from the end of topic which mea Let's produce a single record to the topic. -``` +```bash copy="fl" $ fluvio produce cat-facts {"fact":"A cat’s jaw can’t move sideways, so a cat can’t chew large chunks of food.","length":74} Ok! @@ -157,7 +154,7 @@ Ok! (2 rows) ``` -# Cleaning up +## Clean up Same in previous tutorials, use `cdk deploy shutdown` to stop the connector. @@ -170,7 +167,7 @@ You can combine this tutorial with previous tutorials to create a complete data With Fluvio's event driven architecture, source and sink can be run independently and doesn't effect each other. You can also chain together multiple sources and sinks to create complex data pipelines. -## Reference +## References * [Fluvio CLI Produce] * [Fluvio CLI Consume] @@ -179,6 +176,8 @@ With Fluvio's event driven architecture, source and sink can be run independentl * [SmartModule] * [Transformations] +[Part 1: HTTP Source -> Topic]: http-source.mdx +[Part 2: Connector Transformations]: connector-transformations.mdx [Connector Overview]: connectors/overview.mdx [Fluvio CLI Produce]: fluvio/cli/fluvio/produce.mdx [Fluvio CLI Consume]: fluvio/cli/fluvio/consume.mdx