Update Building System to Support ARM64 Images (#522)

- Add ARM64 image.
- Separate ctl from the release images.
- Add `skywalking-banyandb-test` images for testing.

.github/workflows/e2e.storage.yml

@@ -88,9 +88,11 @@ jobs:
         run: make generate
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
+      - name: Build binary
+        run: make release
       - name: Build docker image
         run: |
-          make docker.build || make docker.build
+          make -C test/docker build | make -C test/docker build
           docker image ls
       - name: ${{ matrix.test.name }}
         uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180

.github/workflows/publish-docker.yml

@@ -75,14 +75,17 @@ jobs:
         run: make generate
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
+      - name: Build Linux binaries
+        run: |
+          TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release
       - name: Build Windows binaries
         run: |
-          GOOS=windows GOARCH=amd64 make -C banyand banyand-server-static
-          GOOS=windows GOARCH=amd64 make -C bydbctl build
+          TARGET_OS=windows PLATFORMS=windows/amd64 make release
       - name: Build docker image
         if: github.ref != 'refs/heads/main' # Only build docker image on PR(Push image when pushed to main branch)
         run: |
           make docker.build || make docker.build
+          make -C test/docker build | make -C test/docker build
           docker image ls
       - name: Log in to the Container registry
         uses: docker/login-action@v1.10.0
@@ -94,4 +97,5 @@ jobs:
       - name: Push docker image
         if: github.ref == 'refs/heads/main'
         run: |
-          make docker.push || make docker.push
+          PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || make docker.push
+          make -C test/docker push | make -C test/docker push

CHANGES.md

@@ -24,6 +24,7 @@ Release Notes.
 - Introduce the round-robin selector to Liaison Node.
 - Optimize query performance of series index.
 - Add liaison, remote queue, storage(rotation), time-series tables, metadata cache and scheduler metrics.
+- Add HTTP health check endpoint for the data node.
 
 ### Bugs
 
@@ -53,6 +54,11 @@ Release Notes.
 ### Chores
 
 Bump up the version of infra e2e framework.
+Separate the monolithic release package into two packages: banyand and bydbctl.
+Separate the monolithic Docker image into two images: banyand and bydbctl.
+Update CI to publish linux/amd64 and linux/arm64 Docker images.
+Make the build system compiles the binary based on the platform which is running on.
+Push "skywalking-banyandb-test" image for e2e and stress test. This image contains bydbctl to do a health check.
 
 ## 0.6.1
 

Makefile

@@ -149,20 +149,12 @@ license-dep: default ## Fix license header issues
 ##@ Docker targets
 
 docker.build: TARGET=docker
-docker.build: DIR=docker
-docker.build:
-	$(MAKE) $(TARGET) -C $(DIR); \
-	if [ $$? -ne 0 ]; then \
-		exit 1; \
-	fi; \
+docker.build: PROJECTS:= banyand bydbctl
+docker.build: default ## Build docker images
 
 docker.push: TARGET=docker.push
-docker.push: DIR=docker
-docker.push: 
-	$(MAKE) $(TARGET) -C $(DIR); \
-	if [ $$? -ne 0 ]; then \
-		exit 1; \
-	fi; \
+docker.push: PROJECTS:= banyand bydbctl
+docker.push: default ## Push docker images
 
 default:
 	@for PRJ in $(PROJECTS); do \
@@ -189,7 +181,8 @@ release-source: clean ## Package source archive
 	${RELEASE_SCRIPTS} -s
 
 release-sign: ## Sign artifacts
-	${RELEASE_SCRIPTS} -k bin
+	${RELEASE_SCRIPTS} -k banyand
+	${RELEASE_SCRIPTS} -k bydbctl
 	${RELEASE_SCRIPTS} -k src
 
 release-assembly: release-binary release-sign ## Generate release package

docker/Dockerfile → banyand/Dockerfile

@@ -14,54 +14,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.22 AS dev
-WORKDIR /app
-ENV GOOS="linux"
-ENV CGO_ENABLED=0
-
-RUN go install github.com/cosmtrek/air@latest \
-    && go install github.com/go-delve/delve/cmd/dlv@latest
-
-EXPOSE 8080
-EXPOSE 2345
-
-ENTRYPOINT ["air"]
-
-FROM golang:1.22 AS base
-
-ENV GOPATH "/go"
-ENV GO111MODULE "on"
-WORKDIR /src
-COPY go.* ./
-RUN go mod download
-
-FROM base AS builder
-
-RUN --mount=target=. \
-            --mount=type=cache,target=/root/.cache/go-build \
-            BUILD_DIR=/out make -C banyand banyand-server-static
-RUN --mount=target=. \
-            --mount=type=cache,target=/root/.cache/go-build \
-            BUILD_DIR=/out make -C bydbctl build
 
 FROM alpine:edge AS certs
 RUN apk add --no-cache ca-certificates && update-ca-certificates
 
-FROM busybox:stable-glibc as build-linux
+FROM busybox:stable-glibc AS build-linux
 
-COPY --from=builder /out/banyand-server-static /banyand
+ARG TARGETARCH
+
+COPY build/bin/linux/${TARGETARCH}/banyand-server-static /banyand
 COPY --from=certs /etc/ssl/certs /etc/ssl/certs
-COPY --from=builder /out/bydbctl /bydbctl
 
-FROM mcr.microsoft.com/windows/servercore:ltsc2022 as build-windows
+FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS build-windows
+
+ARG TARGETARCH
 
-COPY banyand/build/bin/banyand-server-static "/banyand"
-COPY bydbctl/build/bin/bydbctl "/bydbctl"
+COPY build/bin/windows/${TARGETARCH}/banyand-server-static "/banyand"
 
 FROM build-${TARGETOS} AS final
 
 EXPOSE 17912
 EXPOSE 17913
 EXPOSE 6060
+EXPOSE 2121
 
-ENTRYPOINT ["/banyand"]
+ENTRYPOINT ["/banyand"]

banyand/Makefile

@@ -19,15 +19,22 @@
 NAME := banyand
 SERVER := $(NAME)-server
 BINARIES := $(SERVER)
-DEBUG_BINARIES := $(SERVER)-debug
 
 IMG_NAME := skywalking-banyandb
 
 include ../scripts/build/version.mk
 include ../scripts/build/base.mk
 include ../scripts/build/generate_go.mk
 include ../scripts/build/build.mk
+include ../scripts/build/docker.mk
 include ../scripts/build/test.mk
 include ../scripts/build/lint.mk
 include ../scripts/build/vendor.mk
 include ../scripts/build/help.mk
+
+prepare-build: generate
+
+docker.dev:
+	@echo "Building $(IMG) with platform $(PLATFORMS)"
+	@pwd
+	time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) --load --no-cache -t $(IMG) -f Dockerfile.dev --provenance=false .

banyand/liaison/http/server.go

@@ -40,6 +40,7 @@ import (
 	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
 	propertyv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/property/v1"
 	streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
+	"github.com/apache/skywalking-banyandb/pkg/healthcheck"
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/run"
 	"github.com/apache/skywalking-banyandb/ui"
@@ -161,7 +162,7 @@ func (p *server) Serve() run.StopNotify {
 	} else {
 		opts = append(opts, grpc.WithTransportCredentials(p.creds))
 	}
-	client, err := newHealthCheckClient(ctx, p.l, p.grpcAddr, opts)
+	client, err := healthcheck.NewClient(ctx, p.l, p.grpcAddr, opts)
 	if err != nil {
 		p.l.Error().Err(err).Msg("Failed to health check client")
 		close(p.stopCh)