Add GRPC Reflection Service to Enhance Dynamic Discovery (#1328)

Introduces a new GRPC reflection service to the plgd-dev/hub project, enhancing the system's ability to dynamically discover GRPC services and methods. This is particularly useful for developers and tools that need to interact with GRPC services without prior knowledge of their interfaces. ### Key Features: **Service Implementation:** - A dedicated GRPC reflection service is implemented, allowing clients to query available GRPC services and their methods dynamically. - The service is built using a multi-stage Dockerfile to optimize the build process and final image size. **Configuration and Deployment:** - A new configuration file (`grpc-reflection.yaml`) is introduced, detailing service-specific settings such as logging levels and GRPC addresses. - Kubernetes deployment configurations are added, including Helm chart updates to manage the service's lifecycle within a Kubernetes cluster. **Integration with Existing Infrastructure:** - The service is integrated into the existing build and deployment pipelines, ensuring seamless operation alongside other services. - Environment variables and configuration options are provided to customize the service's behavior and integration points. This addition significantly enhances the plgd-dev/hub's capabilities, providing a robust solution for GRPC service discovery and interaction. --------- Co-authored-by: Jozef Kralik <jozef.kralik@plgd.dev>
plgd-dev · Aug 12, 2024 · a0f461c · a0f461c
1 parent 009d774
commit a0f461c
Show file tree

Hide file tree

Showing 46 changed files with 2,211 additions and 29 deletions.
diff --git a/.github/workflows/build-publish.yaml b/.github/workflows/build-publish.yaml
@@ -109,6 +109,9 @@ jobs:
           - name: cert-tool
             directory: tools/cert-tool
             file: tools/cert-tool/Dockerfile
+          - name: grpc-reflection
+            directory: tools/grpc-reflection
+            file: tools/grpc-reflection/Dockerfile
           - name: snippet-service
             directory: snippet-service
             file: .tmp/docker/snippet-service/Dockerfile

diff --git a/bundle/Dockerfile b/bundle/Dockerfile
@@ -120,6 +120,13 @@ RUN go build -ldflags "-linkmode external -extldflags -static -X github.com/plgd
     -o "/go/bin/$service" \
     ./cmd/service
 
+#grpc-reflection
+ARG service=grpc-reflection
+WORKDIR $root_directory/tools/$service
+RUN go build -ldflags "-linkmode external -extldflags -static -X github.com/plgd-dev/hub/v2/pkg/build.CommitDate=$COMMIT_DATE -X github.com/plgd-dev/hub/v2/pkg/build.CommitHash=$SHORT_COMMIT -X github.com/plgd-dev/hub/v2/pkg/build.BuildDate=$DATE -X github.com/plgd-dev/hub/v2/pkg/build.Version=$VERSION -X github.com/plgd-dev/hub/v2/pkg/build.ReleaseURL=$RELEASE_URL" \
+    -o "/go/bin/$service" \
+    ./cmd/service
+
 #nats
 WORKDIR $root_directory
 RUN apkArch="$(apk --print-arch)"; \
@@ -187,6 +194,8 @@ COPY --from=build /go/src/github.com/plgd-dev/hub/bundle/run.sh /usr/local/bin/r
 COPY --from=build /go/src/github.com/plgd-dev/hub/bundle/nginx /nginx
 COPY --from=build /go/bin/snippet-service /usr/local/bin/snippet-service
 COPY --from=build /go/src/github.com/plgd-dev/hub/snippet-service/config.yaml /configs/snippet-service.yaml
+COPY --from=build /go/bin/grpc-reflection /usr/local/bin/grpc-reflection
+COPY --from=build /go/src/github.com/plgd-dev/hub/tools/grpc-reflection/config.yaml /configs/grpc-reflection.yaml
 
 # install scylla
 RUN curl -sSf get.scylladb.com/server | sudo bash -s -- --scylla-version 5.4
@@ -251,6 +260,7 @@ ENV SNIPPET_SERVICE_PORT=9091
 ENV HTTP_SNIPPET_SERVICE_PORT=9092
 ENV M2M_OAUTH_SERVER_PORT=9079
 ENV HTTP_M2M_OAUTH_SERVER_PORT=9080
+ENV GRPC_REFLECTION_PORT=9078
 
 # OAuth
 ENV DEVICE_PROVIDER=plgd

diff --git a/bundle/nginx/nginx.conf.template b/bundle/nginx/nginx.conf.template
@@ -148,6 +148,19 @@ http {
       proxy_set_header Connection $connection_upgrade;
       proxy_set_header Host $host;
     }
+    location ~ ^(/grpc.reflection.v1alpha.ServerReflection|/grpc.reflection.v1.ServerReflection) {
+      set $upstream_grpc_gateway grpcs://127.0.0.1:REPLACE_GRPC_REFLECTION_PORT;
+      grpc_pass                      $upstream_grpc_gateway;
+      grpc_ssl_certificate           /data/certs/internal/endpoint.crt;
+      grpc_ssl_certificate_key       /data/certs/internal/endpoint.key;
+      grpc_ssl_trusted_certificate   /data/certs/root_ca.crt;
+      grpc_read_timeout       31536000s;
+      grpc_send_timeout       31536000s;
+      client_body_timeout     31536000s;
+      grpc_socket_keepalive   on;
+      proxy_ssl_trusted_certificate   /data/certs/root_ca.crt;
+      proxy_ssl_verify                on;
+    }
     location ~ ^/ {
       set $upstream_http_gateway https://127.0.0.1:REPLACE_HTTP_GATEWAY_PORT;
       proxy_pass                     $upstream_http_gateway;

diff --git a/bundle/run.sh b/bundle/run.sh
@@ -28,6 +28,7 @@ export SNIPPET_SERVICE_ADDRESS="localhost:${SNIPPET_SERVICE_PORT}"
 export SNIPPET_SERVICE_HTTP_ADDRESS="localhost:${HTTP_SNIPPET_SERVICE_PORT}"
 export M2M_OAUTH_SERVER_ADDRESS="localhost:${M2M_OAUTH_SERVER_PORT}"
 export M2M_OAUTH_SERVER_HTTP_ADDRESS="localhost:${HTTP_M2M_OAUTH_SERVER_PORT}"
+export GRPC_REFLECTION_ADDRESS="localhost:${GRPC_REFLECTION_PORT}"
 
 export INTERNAL_CERT_DIR_PATH="$CERTIFICATES_PATH/internal"
 export GRPC_INTERNAL_CERT_NAME="endpoint.crt"
@@ -462,6 +463,58 @@ while read -r line; do
   fi
 done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/cloud2cloud-connector.yaml | sort | uniq)
 
+## snippet-service
+### setup root-cas
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+    cp $CA_POOL ${file}
+  fi
+done < <(yq e '[.. | select(has("caPool")) | .caPool]' /configs/snippet-service.yaml | sort | uniq)
+### setup certificates
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $CERT_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("certFile")) | .certFile]' /configs/snippet-service.yaml | sort | uniq)
+### setup private keys
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $KEY_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/snippet-service.yaml| sort | uniq)
+
+## grpc-reflection
+### setup root-cas
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+    cp $CA_POOL ${file}
+  fi
+done < <(yq e '[.. | select(has("caPool")) | .caPool]' /configs/grpc-reflection.yaml | sort | uniq)
+### setup certificates
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $CERT_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("certFile")) | .certFile]' /configs/grpc-reflection.yaml | sort | uniq)
+### setup private keys
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $KEY_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/grpc-reflection.yaml| sort | uniq)
+
 mkdir -p ${OAUTH_KEYS_PATH}
 if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${OAUTH_ID_TOKEN_KEY_PATH}" ]; then
   openssl genrsa -out ${OAUTH_ID_TOKEN_KEY_PATH} 4096
@@ -491,6 +544,7 @@ if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${NGINX_PATH}/nginx.conf" ]; then
   sed -i "s/REPLACE_HTTP_SNIPPET_SERVICE_PORT/$HTTP_SNIPPET_SERVICE_PORT/g" ${NGINX_PATH}/nginx.conf
   sed -i "s/REPLACE_M2M_OAUTH_SERVER_PORT/$M2M_OAUTH_SERVER_PORT/g" ${NGINX_PATH}/nginx.conf
   sed -i "s/REPLACE_HTTP_M2M_OAUTH_SERVER_PORT/$HTTP_M2M_OAUTH_SERVER_PORT/g" ${NGINX_PATH}/nginx.conf
+  sed -i "s/REPLACE_GRPC_REFLECTION_PORT/$GRPC_REFLECTION_PORT/g" ${NGINX_PATH}/nginx.conf
 fi
 
 # nats
@@ -1356,6 +1410,44 @@ while true; do
   sleep 1
 done
 
+# grpc-reflection
+echo "starting grpc-reflection"
+## configuration
+if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "/data/grpc-reflection.yaml" ]; then
+cat /configs/grpc-reflection.yaml |
+  yq e "\
+  .hubID = \"${HUB_ID}\" |
+  .log.level = \"${LOG_LEVEL}\" |
+  .apis.grpc.address = \"${GRPC_REFLECTION_ADDRESS}\" |
+  .clients.openTelemetryCollector.grpc.enabled = ${OPEN_TELEMETRY_EXPORTER_ENABLED} |
+  .clients.openTelemetryCollector.grpc.address = \"${OPEN_TELEMETRY_EXPORTER_ADDRESS}\" |
+  .clients.openTelemetryCollector.grpc.tls.caPool = \"${OPEN_TELEMETRY_EXPORTER_CA_POOL}\" |
+  .clients.openTelemetryCollector.grpc.tls.keyFile = \"${OPEN_TELEMETRY_EXPORTER_KEY_FILE}\" |
+  .clients.openTelemetryCollector.grpc.tls.certFile = \"${OPEN_TELEMETRY_EXPORTER_CERT_FILE}\" |
+  .clients.openTelemetryCollector.grpc.tls.useSystemCAPool = true
+" - > /data/grpc-reflection.yaml
+fi
+grpc-reflection --config /data/grpc-reflection.yaml >$LOGS_PATH/grpc-reflection.log 2>&1 &
+status=$?
+grpc_reflection_pid=$!
+if [ $status -ne 0 ]; then
+  echo "Failed to start grpc-reflection: $status"
+  sync
+  cat $LOGS_PATH/grpc-reflection.log
+  exit $status
+fi
+
+# waiting for ca. Without wait, sometimes the service didn't connect.
+i=0
+while true; do
+  i=$((i+1))
+  if openssl s_client -connect ${GRPC_REFLECTION_ADDRESS} -cert ${INTERNAL_CERT_DIR_PATH}/${GRPC_INTERNAL_CERT_NAME} -key ${INTERNAL_CERT_DIR_PATH}/${GRPC_INTERNAL_CERT_KEY_NAME} <<< "Q" 2>/dev/null > /dev/null; then
+    break
+  fi
+  echo "Try to reconnect to grpc-reflection(${GRPC_REFLECTION_ADDRESS}) $i"
+  sleep 1
+done
+
 echo "Open browser at https://${DOMAIN}"
 
 # Naive check runs checks once a minute to see if either of the processes exited.
@@ -1487,4 +1579,11 @@ while sleep 10; do
     cat $LOGS_PATH/snippet-service.log
    exit 1
   fi
+  ps aux |grep $grpc_reflection_pid |grep -q -v grep
+  if [ $? -ne 0 ]; then
+    echo "grpc-reflection has already exited."
+    sync
+    cat $LOGS_PATH/grpc-reflection.log
+   exit 1
+  fi
 done
diff --git a/certificate-authority/Makefile b/certificate-authority/Makefile
@@ -49,6 +49,7 @@ GRPCGATEWAY_MODULE_PATH := $(shell go list -m -f '{{.Dir}}' github.com/grpc-ecos
 proto/generate:
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/cert.proto
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/signingRecords.proto
+	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/service.proto
 	protoc-go-inject-tag -input=$(WORKING_DIRECTORY)/pb/signingRecords.pb.go
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --openapiv2_out=$(REPOSITORY_DIRECTORY) \
 		--openapiv2_opt logtostderr=true \