Compaction foundations

.github/workflows/_rust-ci-base.yaml

@@ -0,0 +1,105 @@
+name: CI Base
+on:
+  workflow_call:
+    inputs:
+      project:
+        type: string
+        required: true
+        description: 'The Rust project to target'
+      dir:
+        type: string
+        required: false
+        default: '.'
+        description: 'The directory to start from'
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ${{ inputs.dir }}/${{ inputs.project }}/target
+          key: cargo-${{ hashFiles(format('{0}/{1}/Cargo.lock', inputs.dir, inputs.project)) }}
+          restore-keys: |
+            ${{ inputs.project }}-cargo-
+
+      - name: Setup | Toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          default: true
+          components: clippy,rustfmt
+
+      - name: Build | Clippy
+        uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --manifest-path ${{ inputs.dir }}/${{ inputs.project }}/Cargo.toml --all-targets -- -D warnings
+
+      - name: Build | Rustfmt
+        run: cargo fmt --manifest-path ${{ inputs.dir }}/${{ inputs.project }}/Cargo.toml -- --check
+
+  check:
+    name: check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup | Cache Cargo
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ${{ inputs.dir }}/${{ inputs.project }}/target
+          key: cargo-${{ hashFiles(format('{0}/{1}/Cargo.lock', inputs.dir, inputs.project)) }}
+          restore-keys: |
+            ${{ inputs.project }}-cargo-
+
+      - name: Setup | Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Build | Check
+        run: cargo check --all --manifest-path ${{ inputs.dir }}/${{ inputs.project }}/Cargo.toml
+
+  test:
+    name: test
+    needs: check # Ensure check is run first.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup | Cache Cargo
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            ${{ inputs.dir }}/${{ inputs.project }}/target
+          key: cargo-${{ hashFiles(format('{0}/{1}/Cargo.lock', inputs.dir, inputs.project)) }}
+          restore-keys: |
+            ${{ inputs.project }}-cargo-
+
+      - name: Setup | Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Build | Test
+        run: cargo test --manifest-path ${{ inputs.dir }}/${{ inputs.project }}/Cargo.toml
+
+      - name: Build | Debug
+        run: cargo build --manifest-path ${{ inputs.dir }}/${{ inputs.project }}/Cargo.toml

.github/workflows/ci-cli.yaml

@@ -0,0 +1,16 @@
+name: CI CLI
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'hadron-cli/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  ci:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: hadron-cli

.github/workflows/ci-client.yaml

@@ -0,0 +1,16 @@
+name: CI Client
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'hadron-client/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  ci:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: hadron-client

.github/workflows/ci-core.yaml

@@ -0,0 +1,16 @@
+name: CI Core
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'hadron-core/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  ci:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: hadron-core

.github/workflows/ci-examples.yaml

@@ -0,0 +1,22 @@
+name: CI Core
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'examples/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  pipeline-txp:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: pipeline-transactional-processing
+      dir: examples
+  pipeline-txp:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: stream-transactional-processing
+      dir: examples

.github/workflows/ci-guide.yaml

@@ -0,0 +1,22 @@
+name: CI Guide
+on:
+  push:
+    branches:
+      - *
+    paths:
+      - guide/**
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup | Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup mdBook
+        uses: peaceiris/actions-mdbook@v1
+        with:
+          mdbook-version: '0.4.6'
+
+      - name: Build
+        run: mdbook build guide

.github/workflows/ci-operator.yaml

@@ -0,0 +1,16 @@
+name: CI Operator
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'hadron-operator/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  ci:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: hadron-operator

.github/workflows/ci-stream.yaml

@@ -0,0 +1,16 @@
+name: CI Stream
+on:
+  push:
+    branches:
+      - "*"
+    paths:
+      - 'hadron-stream/**'
+      - "!**.md"
+    tags-ignore:
+      - "*"
+
+jobs:
+  ci:
+    uses: hadron-project/hadron/.github/workflows/_rust-ci-base.yaml@99-compaction
+    with:
+      project: hadron-stream

Justfile

@@ -20,8 +20,8 @@ kindLoadCli tag="latest":
     kind load docker-image --name hadron ghcr.io/hadron-project/hadron/hadron-cli:{{tag}}
 
 # Run the Hadron CLI within the kind cluster.
-runCli tag="latest" pullPolicy="IfNotPresent":
-    kubectl --context="kind-hadron" run hadron-cli --rm -it \
+runCli num="0" tag="latest" pullPolicy="IfNotPresent":
+    kubectl --context="kind-hadron" run hadron-cli-{{num}} --rm -it \
         --env HADRON_TOKEN=$(kubectl get secret hadron-full-access -o=jsonpath='{.data.token}' | base64 --decode) \
         --env HADRON_URL="http://events.default.svc.cluster.local:7000" \
         --image ghcr.io/hadron-project/hadron/hadron-cli:{{tag}} --image-pull-policy={{pullPolicy}}

charts/hadron-operator/crds/stream.yaml

@@ -21,13 +21,16 @@ spec:
         - jsonPath: ".spec.debug"
           name: Debug
           type: boolean
-        - jsonPath: ".spec.pvc_volume_size"
+        - jsonPath: ".spec.retentionPolicy"
+          name: Retention Policy
+          type: string
+        - jsonPath: ".spec.pvcVolumeSize"
           name: PVC Volume Size
           type: string
-        - jsonPath: ".spec.pvc_access_modes"
+        - jsonPath: ".spec.pvcAccessModes"
           name: PVC Access Modes
           type: string
-        - jsonPath: ".spec.pvc_storage_class"
+        - jsonPath: ".spec.pvcStorageClass"
           name: PVC Storage Class
           type: string
       name: v1beta1
@@ -63,6 +66,27 @@ spec:
                 pvcVolumeSize:
                   description: "The volume size to use for the Stream's backing StatefulSet PVCs."
                   type: string
+                retentionPolicy:
+                  default:
+                    retentionSeconds: 604800
+                    strategy: time
+                  description: The retention policy to use for data on the Stream.
+                  properties:
+                    retentionSeconds:
+                      description: "For `Time` retention policy, this specifies the amount of time to retain data on the Stream in seconds."
+                      format: uint64
+                      minimum: 0.0
+                      nullable: true
+                      type: integer
+                    strategy:
+                      description: Retention policy to use.
+                      enum:
+                        - retain
+                        - time
+                      type: string
+                  required:
+                    - strategy
+                  type: object
               required:
                 - image
                 - partitions