From d82921e3bcf369b3da08761949312e190ad4fdee Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Sun, 31 Mar 2024 19:14:20 -0400 Subject: [PATCH 1/2] apiserver: terminate auth server TLS at the apiproxy This makes our deployment more consistent and allows envoy to rate limit traffic to the auth server too. Well can also use a single cert with multiple DNS names instead of 3 TLS certs. Signed-off-by: Hiram Chirino --- .../base/apiproxy/certificate-api.yaml | 23 ----- .../base/apiproxy/certificate-frontend.yaml | 23 ----- .../base/{auth => apiproxy}/certificate.yaml | 6 +- deploy/nexodus/base/apiproxy/deployment.yaml | 20 ++-- deploy/nexodus/base/apiproxy/files/envoy.yaml | 98 ++++++++++++++++--- .../base/apiproxy/files/sds_config.yaml | 18 +--- .../ingress-auth.yaml} | 2 +- .../nexodus/base/apiproxy/kustomization.yaml | 9 +- deploy/nexodus/base/apiserver/deployment.yaml | 2 +- .../nexodus/base/apiserver/kustomization.yaml | 2 +- deploy/nexodus/base/auth/deployment.yaml | 28 ------ deploy/nexodus/base/auth/kustomization.yaml | 2 - deploy/nexodus/base/auth/service.yaml | 4 +- .../overlays/openshift/kustomization.yaml | 16 +-- .../overlays/playground/kustomization.yaml | 24 ++--- .../nexodus/overlays/prod/kustomization.yaml | 26 ++--- deploy/nexodus/overlays/qa/kustomization.yaml | 25 ++--- 17 files changed, 135 insertions(+), 193 deletions(-) delete mode 100644 deploy/nexodus/base/apiproxy/certificate-api.yaml delete mode 100644 deploy/nexodus/base/apiproxy/certificate-frontend.yaml rename deploy/nexodus/base/{auth => apiproxy}/certificate.yaml (75%) rename deploy/nexodus/base/{auth/ingress.yaml => apiproxy/ingress-auth.yaml} (93%) diff --git a/deploy/nexodus/base/apiproxy/certificate-api.yaml b/deploy/nexodus/base/apiproxy/certificate-api.yaml deleted file mode 100644 index 098102edb..000000000 --- a/deploy/nexodus/base/apiproxy/certificate-api.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: nexodus-api-cert -spec: - secretName: nexodus-api-cert - duration: 2160h0m0s - renewBefore: 360h0m0s - subject: - organizations: - - nexodus - privateKey: - algorithm: RSA - encoding: PKCS1 - size: 2048 - usages: - - server auth - - client auth - dnsNames: - - api.try.nexodus.127.0.0.1.nip.io - issuerRef: - name: nexodus-issuer - kind: Issuer diff --git a/deploy/nexodus/base/apiproxy/certificate-frontend.yaml b/deploy/nexodus/base/apiproxy/certificate-frontend.yaml deleted file mode 100644 index 640256428..000000000 --- a/deploy/nexodus/base/apiproxy/certificate-frontend.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: nexodus-frontend-cert -spec: - secretName: nexodus-frontend-cert - duration: 2160h0m0s - renewBefore: 360h0m0s - subject: - organizations: - - nexodus - privateKey: - algorithm: RSA - encoding: PKCS1 - size: 2048 - usages: - - server auth - - client auth - dnsNames: - - try.nexodus.127.0.0.1.nip.io - issuerRef: - name: nexodus-issuer - kind: Issuer diff --git a/deploy/nexodus/base/auth/certificate.yaml b/deploy/nexodus/base/apiproxy/certificate.yaml similarity index 75% rename from deploy/nexodus/base/auth/certificate.yaml rename to deploy/nexodus/base/apiproxy/certificate.yaml index edbf9f9f9..e5d87d8a8 100644 --- a/deploy/nexodus/base/auth/certificate.yaml +++ b/deploy/nexodus/base/apiproxy/certificate.yaml @@ -1,9 +1,9 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: nexodus-auth-cert + name: nexodus-cert spec: - secretName: nexodus-auth-cert + secretName: nexodus-cert duration: 2160h0m0s renewBefore: 360h0m0s subject: @@ -17,6 +17,8 @@ spec: - server auth - client auth dnsNames: + - try.nexodus.127.0.0.1.nip.io + - api.try.nexodus.127.0.0.1.nip.io - auth.try.nexodus.127.0.0.1.nip.io issuerRef: name: nexodus-issuer diff --git a/deploy/nexodus/base/apiproxy/deployment.yaml b/deploy/nexodus/base/apiproxy/deployment.yaml index 921c2cfbc..d7ff2dde4 100644 --- a/deploy/nexodus/base/apiproxy/deployment.yaml +++ b/deploy/nexodus/base/apiproxy/deployment.yaml @@ -37,6 +37,11 @@ spec: configMapKeyRef: name: apiproxy key: APIPROXY_WEB_DOMAIN + - name: APIPROXY_AUTH_DOMAIN + valueFrom: + configMapKeyRef: + name: apiproxy + key: APIPROXY_AUTH_DOMAIN - name: APIPROXY_WEB_ORIGINS valueFrom: configMapKeyRef: @@ -92,11 +97,8 @@ spec: mountPath: /configs/envoy - name: envoy-unix-sockets mountPath: /sockets - - name: nexodus-api-cert - mountPath: "/configs/envoy-certs/nexodus-api-cert" - readOnly: true - - name: nexodus-frontend-cert - mountPath: "/configs/envoy-certs/nexodus-frontend-cert" + - name: nexodus-cert + mountPath: "/configs/envoy-certs/nexodus-cert" readOnly: true readinessProbe: @@ -130,11 +132,7 @@ spec: - name: envoy-unix-sockets emptyDir: medium: Memory - - name: nexodus-api-cert - secret: - secretName: nexodus-api-cert - optional: false - - name: nexodus-frontend-cert + - name: nexodus-cert secret: - secretName: nexodus-frontend-cert + secretName: nexodus-cert optional: false diff --git a/deploy/nexodus/base/apiproxy/files/envoy.yaml b/deploy/nexodus/base/apiproxy/files/envoy.yaml index c606ae2f9..c653429d1 100644 --- a/deploy/nexodus/base/apiproxy/files/envoy.yaml +++ b/deploy/nexodus/base/apiproxy/files/envoy.yaml @@ -31,7 +31,9 @@ static_resources: - name: auth connect_timeout: 5s - type: LOGICAL_DNS + type: STRICT_DNS + dns_refresh_rate: 1s + dns_lookup_family: V4_ONLY lb_policy: ROUND_ROBIN load_assignment: cluster_name: auth @@ -41,11 +43,7 @@ static_resources: address: socket_address: address: auth - port_value: 8443 - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + port_value: 8080 # upstream server: apiserver - name: apiserver @@ -196,6 +194,82 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector filter_chains: + - filter_chain_match: + server_names: ["${APIPROXY_AUTH_DOMAIN}"] + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + # TODO: try to get dynamic loading of certs working... + # tls_certificate_sds_secret_configs: + # name: nexodus-cert + # sds_config: + # path: /configs/envoy/sds_config.yaml + tls_certificates: + - certificate_chain: + filename: "/configs/envoy-certs/nexodus-cert/tls.crt" + private_key: + filename: "/configs/envoy-certs/nexodus-cert/tls.key" + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + use_remote_address: true + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + + stat_prefix: auth + http_filters: + + # This is needed to enable the rate limiter: + - name: envoy.filters.http.ratelimit + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit + # domain: "%REQ(:authority)%" + domain: "nexodus" + failure_mode_deny: false + timeout: 0.5s + enable_x_ratelimit_headers: DRAFT_VERSION_03 + rate_limit_service: + transport_api_version: V3 + grpc_service: + envoy_grpc: + cluster_name: ratelimiter + + # This is mandatory in order to have the HTTP routes above. + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + + route_config: + name: auth + virtual_hosts: + - name: auth + domains: + - "${APIPROXY_AUTH_DOMAIN}" + retry_policy: + num_retries: 2 + retry_back_off: + base_interval: 0.25s + max_interval: 60s + retry_on: 5xx,connect-failure,refused-stream + routes: + - name: default + match: + prefix: / + route: + timeout: 10s + cluster: auth + rate_limits: + - actions: + - generic_key: + descriptor_key: resource_group + descriptor_value: auth + - filter_chain_match: server_names: ["${APIPROXY_WEB_DOMAIN}"] transport_socket: @@ -205,14 +279,14 @@ static_resources: common_tls_context: # TODO: try to get dynamic loading of certs working... # tls_certificate_sds_secret_configs: - # name: nexodus-frontend-cert + # name: nexodus-cert # sds_config: # path: /configs/envoy/sds_config.yaml tls_certificates: - certificate_chain: - filename: "/configs/envoy-certs/nexodus-frontend-cert/tls.crt" + filename: "/configs/envoy-certs/nexodus-cert/tls.crt" private_key: - filename: "/configs/envoy-certs/nexodus-frontend-cert/tls.key" + filename: "/configs/envoy-certs/nexodus-cert/tls.key" filters: - name: envoy.filters.network.http_connection_manager typed_config: @@ -284,14 +358,14 @@ static_resources: common_tls_context: # TODO: try to get dynamic loading of certs working... # tls_certificate_sds_secret_configs: - # name: nexodus-api-cert + # name: nexodus-cert # sds_config: # path: /configs/envoy/sds_config.yaml tls_certificates: - certificate_chain: - filename: "/configs/envoy-certs/nexodus-api-cert/tls.crt" + filename: "/configs/envoy-certs/nexodus-cert/tls.crt" private_key: - filename: "/configs/envoy-certs/nexodus-api-cert/tls.key" + filename: "/configs/envoy-certs/nexodus-cert/tls.key" filters: - name: envoy.filters.network.http_connection_manager typed_config: diff --git a/deploy/nexodus/base/apiproxy/files/sds_config.yaml b/deploy/nexodus/base/apiproxy/files/sds_config.yaml index 34939e602..3d538520c 100644 --- a/deploy/nexodus/base/apiproxy/files/sds_config.yaml +++ b/deploy/nexodus/base/apiproxy/files/sds_config.yaml @@ -2,21 +2,11 @@ resources: - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" - name: nexodus-api-cert + name: nexodus-cert tls_certificate: certificate_chain: - filename: /configs/envoy-certs/nexodus-api-cert/tls.crt + filename: /configs/envoy-certs/nexodus-cert/tls.crt private_key: - filename: /configs/envoy-certs/nexodus-api-cert/tls.key + filename: /configs/envoy-certs/nexodus-cert/tls.key watched_directory: - path: /configs/envoy-certs/nexodus-api-cert - - - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" - name: nexodus-frontend-cert - tls_certificate: - certificate_chain: - filename: /configs/envoy-certs/nexodus-frontend-cert/tls.crt - private_key: - filename: /configs/envoy-certs/nexodus-frontend-cert/tls.key - watched_directory: - path: /configs/envoy-certs/nexodus-frontend-cert + path: /configs/envoy-certs/nexodus-cert diff --git a/deploy/nexodus/base/auth/ingress.yaml b/deploy/nexodus/base/apiproxy/ingress-auth.yaml similarity index 93% rename from deploy/nexodus/base/auth/ingress.yaml rename to deploy/nexodus/base/apiproxy/ingress-auth.yaml index cda002b7d..7d68dbc02 100644 --- a/deploy/nexodus/base/auth/ingress.yaml +++ b/deploy/nexodus/base/apiproxy/ingress-auth.yaml @@ -14,6 +14,6 @@ spec: path: "/" backend: service: - name: auth + name: apiproxy port: number: 8443 diff --git a/deploy/nexodus/base/apiproxy/kustomization.yaml b/deploy/nexodus/base/apiproxy/kustomization.yaml index e43c770e4..79b8aa759 100644 --- a/deploy/nexodus/base/apiproxy/kustomization.yaml +++ b/deploy/nexodus/base/apiproxy/kustomization.yaml @@ -4,10 +4,11 @@ kind: Kustomization configMapGenerator: - literals: - APIPROXY_OIDC_URL=https://auth.try.nexodus.127.0.0.1.nip.io/realms/nexodus - - APIPROXY_OIDC_BACKCHANNEL=https://auth:8443/realms/nexodus - - APIPROXY_API_DOMAIN=api.try.nexodus.127.0.0.1.nip.io + - APIPROXY_OIDC_BACKCHANNEL=https://auth:8080/realms/nexodus - APIPROXY_API_URL=https://api.try.nexodus.127.0.0.1.nip.io + - APIPROXY_API_DOMAIN=api.try.nexodus.127.0.0.1.nip.io - APIPROXY_WEB_DOMAIN=try.nexodus.127.0.0.1.nip.io + - APIPROXY_AUTH_DOMAIN=auth.try.nexodus.127.0.0.1.nip.io - APIPROXY_WEB_ORIGINS=https://try.nexodus.127.0.0.1.nip.io - ENVOY_COMP_LOG_LEVEL=upstream:info,http:debug,router:debug,jwt:debug name: apiproxy @@ -19,10 +20,10 @@ resources: - deployment.yaml - ingress-api.yaml - ingress-frontend.yaml + - ingress-auth.yaml - service.yaml - service-lb.yaml - - certificate-api.yaml - - certificate-frontend.yaml + - certificate.yaml labels: - includeSelectors: true pairs: diff --git a/deploy/nexodus/base/apiserver/deployment.yaml b/deploy/nexodus/base/apiserver/deployment.yaml index 7057e1485..6e012b076 100644 --- a/deploy/nexodus/base/apiserver/deployment.yaml +++ b/deploy/nexodus/base/apiserver/deployment.yaml @@ -156,7 +156,7 @@ spec: - name: NEXAPI_TLS_KEY valueFrom: secretKeyRef: - name: nexodus-api-cert + name: nexodus-cert key: tls.key - name: NEXAPI_SMTP_FROM valueFrom: diff --git a/deploy/nexodus/base/apiserver/kustomization.yaml b/deploy/nexodus/base/apiserver/kustomization.yaml index 8ea36281c..59625df56 100644 --- a/deploy/nexodus/base/apiserver/kustomization.yaml +++ b/deploy/nexodus/base/apiserver/kustomization.yaml @@ -6,7 +6,7 @@ configMapGenerator: - NEXAPI_DEBUG=1 - NEXAPI_IPAM_URL=http://ipam:9090 - NEXAPI_OIDC_URL=https://auth.try.nexodus.127.0.0.1.nip.io/realms/nexodus - - NEXAPI_OIDC_BACKCHANNEL=https://auth:8443/realms/nexodus + - NEXAPI_OIDC_BACKCHANNEL=http://auth:8080/realms/nexodus - NEXAPI_INSECURE_TLS=1 - NEXAPI_TRACE_ENDPOINT_OTLP="tempo.nexodus-monitoring.svc:4317" - NEXAPI_TRACE_INSECURE="1" diff --git a/deploy/nexodus/base/auth/deployment.yaml b/deploy/nexodus/base/auth/deployment.yaml index 2ad685a92..b9e7ee909 100644 --- a/deploy/nexodus/base/auth/deployment.yaml +++ b/deploy/nexodus/base/auth/deployment.yaml @@ -50,12 +50,6 @@ spec: value: "true" - name: KC_HTTP_PORT value: "8080" - - name: KC_HTTPS_PORT - value: "8443" - - name: KC_HTTPS_CERTIFICATE_FILE - value: "/etc/pki/tls/certs/cert.pem" - - name: KC_HTTPS_CERTIFICATE_KEY_FILE - value: "/etc/pki/tls/private/cert.key" - name: KEYCLOAK_ADMIN value: "admin" - name: KEYCLOAK_ADMIN_PASSWORD @@ -110,19 +104,11 @@ spec: ports: - name: http containerPort: 8080 - - name: https - containerPort: 8443 readinessProbe: httpGet: path: /realms/nexodus port: 8080 volumeMounts: - - name: certificate - mountPath: "/etc/pki/tls/certs" - readOnly: true - - name: privatekey - mountPath: "/etc/pki/tls/private" - readOnly: true - name: realm mountPath: "/opt/keycloak/data/import" readOnly: true @@ -130,20 +116,6 @@ spec: args: ["start", "--import-realm", "--features=preview", "--features=declarative-user-profile"] restartPolicy: Always volumes: - - name: certificate - secret: - secretName: nexodus-auth-cert - items: - - key: tls.crt - path: cert.pem - optional: false - - name: privatekey - secret: - secretName: nexodus-auth-cert - items: - - key: tls.key - path: cert.key - optional: false - name: realm configMap: name: realm diff --git a/deploy/nexodus/base/auth/kustomization.yaml b/deploy/nexodus/base/auth/kustomization.yaml index 7591e00fb..b24904aff 100644 --- a/deploy/nexodus/base/auth/kustomization.yaml +++ b/deploy/nexodus/base/auth/kustomization.yaml @@ -1,9 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - certificate.yaml - deployment.yaml - - ingress.yaml - service.yaml configMapGenerator: - literals: diff --git a/deploy/nexodus/base/auth/service.yaml b/deploy/nexodus/base/auth/service.yaml index f959a6c34..9a143b91f 100644 --- a/deploy/nexodus/base/auth/service.yaml +++ b/deploy/nexodus/base/auth/service.yaml @@ -4,5 +4,5 @@ metadata: name: auth spec: ports: - - port: 8443 - targetPort: 8443 + - port: 8080 + targetPort: 8080 diff --git a/deploy/nexodus/overlays/openshift/kustomization.yaml b/deploy/nexodus/overlays/openshift/kustomization.yaml index e8442381b..ef587aa76 100644 --- a/deploy/nexodus/overlays/openshift/kustomization.yaml +++ b/deploy/nexodus/overlays/openshift/kustomization.yaml @@ -21,21 +21,7 @@ patches: # Use the letsencrypt issuer instead of the self-signed issuer - target: kind: Certificate - name: nexodus-auth-cert - patch: |- - - op: replace - path: /spec/issuerRef/name - value: letsencrypt - - target: - kind: Certificate - name: nexodus-frontend-cert - patch: |- - - op: replace - path: /spec/issuerRef/name - value: letsencrypt - - target: - kind: Certificate - name: nexodus-api-cert + name: nexodus-cert patch: |- - op: replace path: /spec/issuerRef/name diff --git a/deploy/nexodus/overlays/playground/kustomization.yaml b/deploy/nexodus/overlays/playground/kustomization.yaml index d16330b1e..b75d7b723 100644 --- a/deploy/nexodus/overlays/playground/kustomization.yaml +++ b/deploy/nexodus/overlays/playground/kustomization.yaml @@ -23,6 +23,7 @@ configMapGenerator: - APIPROXY_OIDC_URL=https://auth.playground.nexodus.io/realms/nexodus - APIPROXY_API_DOMAIN=api.playground.nexodus.io - APIPROXY_WEB_DOMAIN=playground.nexodus.io + APIPROXY_AUTH_DOMAIN=auth.playground.nexodus.io - APIPROXY_WEB_ORIGINS=https://playground.nexodus.io - ENVOY_COMP_LOG_LEVEL=upstream:info,http:info,router:info,jwt:info - behavior: merge @@ -42,25 +43,14 @@ patches: # Update the dns names for the certificates - target: kind: Certificate - name: nexodus-frontend-cert + name: nexodus-cert patch: |- - op: replace - path: /spec/dnsNames/0 - value: playground.nexodus.io - - target: - kind: Certificate - name: nexodus-api-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: api.playground.nexodus.io - - target: - kind: Certificate - name: nexodus-auth-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: auth.playground.nexodus.io + path: /spec/dnsNames + value: + - playground.nexodus.io + - api.playground.nexodus.io + - auth.playground.nexodus.io # Update the dns names for the ingress - patch: |- - op: replace diff --git a/deploy/nexodus/overlays/prod/kustomization.yaml b/deploy/nexodus/overlays/prod/kustomization.yaml index b8c4c3a31..e145699dc 100644 --- a/deploy/nexodus/overlays/prod/kustomization.yaml +++ b/deploy/nexodus/overlays/prod/kustomization.yaml @@ -22,6 +22,7 @@ configMapGenerator: - APIPROXY_OIDC_URL=https://auth.try.nexodus.io/realms/nexodus - APIPROXY_API_DOMAIN=api.try.nexodus.io - APIPROXY_WEB_DOMAIN=try.nexodus.io + APIPROXY_AUTH_DOMAIN=auth.try.nexodus.io - APIPROXY_WEB_ORIGINS=https://try.nexodus.io - ENVOY_COMP_LOG_LEVEL=upstream:info,http:info,router:info,jwt:info name: apiproxy @@ -40,27 +41,14 @@ patches: # Update the dns names for the certificates - target: kind: Certificate - name: nexodus-frontend-cert + name: nexodus-cert patch: |- - op: replace - path: /spec/dnsNames/0 - value: try.nexodus.io - - target: - kind: Certificate - name: nexodus-api-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: api.try.nexodus.io - - target: - kind: Certificate - name: nexodus-auth-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: auth.try.nexodus.io - - + path: /spec/dnsNames + value: + - try.nexodus.io + - api.try.nexodus.io + - auth.try.nexodus.io - patch: |- - op: replace path: /spec/rules/0/host diff --git a/deploy/nexodus/overlays/qa/kustomization.yaml b/deploy/nexodus/overlays/qa/kustomization.yaml index 2511f502d..5ba162578 100644 --- a/deploy/nexodus/overlays/qa/kustomization.yaml +++ b/deploy/nexodus/overlays/qa/kustomization.yaml @@ -22,6 +22,7 @@ configMapGenerator: - APIPROXY_OIDC_URL=https://auth.qa.nexodus.io/realms/nexodus - APIPROXY_API_DOMAIN=api.qa.nexodus.io - APIPROXY_WEB_DOMAIN=qa.nexodus.io + APIPROXY_AUTH_DOMAIN=auth.qa.nexodus.io - APIPROXY_WEB_ORIGINS=https://qa.nexodus.io - ENVOY_COMP_LOG_LEVEL=upstream:info,http:debug,router:debug,jwt:debug name: apiproxy @@ -38,26 +39,14 @@ patches: # Update the dns names for the certificates - target: kind: Certificate - name: nexodus-frontend-cert + name: nexodus-cert patch: |- - op: replace - path: /spec/dnsNames/0 - value: qa.nexodus.io - - target: - kind: Certificate - name: nexodus-api-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: api.qa.nexodus.io - - target: - kind: Certificate - name: nexodus-auth-cert - patch: |- - - op: replace - path: /spec/dnsNames/0 - value: auth.qa.nexodus.io - + path: /spec/dnsNames + value: + - qa.nexodus.io + - api.qa.nexodus.io + - auth.qa.nexodus.io - patch: |- - op: replace path: /spec/rules/0/host From 77f958c18cf6a8af6f79af8b4ceba41504883889 Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Sat, 30 Mar 2024 11:56:44 -0400 Subject: [PATCH 2/2] Add a docker-compose configuration that can start the apiserver. Also add the `dlv` debugger to the container images so that they can be remotely debugged if needed. Signed-off-by: Hiram Chirino --- Containerfile.apiserver | 6 +- Containerfile.nexd | 3 + Makefile | 1 + contrib/docker-compose/.gitignore | 3 + contrib/docker-compose/README.md | 29 ++++ contrib/docker-compose/configure.sh | 39 +++++ contrib/docker-compose/docker-compose.yml | 197 ++++++++++++++++++++++ contrib/docker-compose/example.env | 78 +++++++++ 8 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 contrib/docker-compose/.gitignore create mode 100644 contrib/docker-compose/README.md create mode 100755 contrib/docker-compose/configure.sh create mode 100644 contrib/docker-compose/docker-compose.yml create mode 100644 contrib/docker-compose/example.env diff --git a/Containerfile.apiserver b/Containerfile.apiserver index 02f2764b4..4a0674b62 100644 --- a/Containerfile.apiserver +++ b/Containerfile.apiserver @@ -21,7 +21,9 @@ RUN dnf update -qy && \ dnf clean all -y &&\ rm -rf /var/cache/yum +RUN go install github.com/go-delve/delve/cmd/dlv@latest +ARG BUILD_PROFILE=dev ARG NEXODUS_PPROF= ARG NEXODUS_RACE_DETECTOR= @@ -33,10 +35,12 @@ COPY . . RUN NOISY_BUILD=y \ NEXODUS_RACE_DETECTOR=${NEXODUS_RACE_DETECTOR} \ NEXODUS_PPROF=${NEXODUS_PPROF} \ + NEXODUS_BUILD_PROFILE=$BUILD_PROFILE \ make dist/apiserver FROM registry.access.redhat.com/ubi8/ubi COPY --from=build /src/dist/apiserver /apiserver +COPY --chmod=755 --from=build /go/bin/dlv /dlv EXPOSE 8080 -ENTRYPOINT [ "/apiserver" ] +CMD ["/apiserver"] \ No newline at end of file diff --git a/Containerfile.nexd b/Containerfile.nexd index edecda1e2..03516775c 100644 --- a/Containerfile.nexd +++ b/Containerfile.nexd @@ -20,6 +20,8 @@ RUN dnf update -qy && \ dnf clean all -y &&\ rm -rf /var/cache/yum +RUN go install github.com/go-delve/delve/cmd/dlv@latest + ARG BUILD_PROFILE=dev ARG NEXODUS_PPROF= ARG NEXODUS_RACE_DETECTOR= @@ -88,6 +90,7 @@ COPY --chmod=755 --from=build /src/dist/nexd /bin/nexd COPY --chmod=755 --from=build /src/dist/nexd-kstore /bin/nexd-kstore COPY --chmod=755 --from=build /src/dist/nexctl /bin/nexctl COPY --chmod=755 --from=build /go/bin/mkcert /bin/mkcert +COPY --chmod=755 --from=build /go/bin/dlv /bin/dlv COPY --chmod=755 --from=build /go/bin/wireguard /bin/nexd-wireguard-go COPY --chmod=755 --from=build /src/udping /bin/udping COPY --chmod=755 --from=build /src/udpong /bin/udpong diff --git a/Makefile b/Makefile index eb6e965b3..b0c2775a1 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ NEXODUS_BUILD_PROFILE?=dev NEXODUS_LDFLAGS:=$(NEXODUS_LDFLAGS) -X main.Version=$(NEXODUS_VERSION)-$(NEXODUS_RELEASE) ifeq ($(NEXODUS_BUILD_PROFILE),dev) NEXODUS_LDFLAGS+=-X main.DefaultServiceURL=https://try.nexodus.127.0.0.1.nip.io + NEXODUS_GCFLAGS+=all=-N -l else ifeq ($(NEXODUS_BUILD_PROFILE),qa) NEXODUS_LDFLAGS+=-X main.DefaultServiceURL=https://qa.nexodus.io else ifeq ($(NEXODUS_BUILD_PROFILE),prod) diff --git a/contrib/docker-compose/.gitignore b/contrib/docker-compose/.gitignore new file mode 100644 index 000000000..171361bd7 --- /dev/null +++ b/contrib/docker-compose/.gitignore @@ -0,0 +1,3 @@ +volumes/ +.env +.env-keys \ No newline at end of file diff --git a/contrib/docker-compose/README.md b/contrib/docker-compose/README.md new file mode 100644 index 000000000..d61738a6a --- /dev/null +++ b/contrib/docker-compose/README.md @@ -0,0 +1,29 @@ +# Running the API server in Docker with Docker Compose + +If you want to run the API server in Docker instead of Kubernetes, you can use the Docker Compose configuration found in this directory. This is great option if your doing local development and don't want to install Kubernetes. + +Firstly you need to start a shell in the `contrib/docker-compose` directory: + +```bash +cd contrib/docker-compose +``` + +Then you need to generate some of the configuration files with: + +```bash +./generate-config.sh +``` + +That wille create the .env and ./volumes directories. Advanced users can modify the .env file to change the configuration of the API server. + +Then you can start the server with: + +```bash +docker-compose up -d +``` + +## Warning: Remote debugging is enabled by default + +The apiserver is started with a go debugger listening on port 2345. You can connect to it with your IDE or a debugger on `localhost:2345`. + +To disable the debugger, you can edit the `docker-compose.yml` file and comment out th line that reads: `command: /dlv --continue --listen=:2345 --api-version=2 --only-same-user=false --headless --accept-multiclient exec /apiserver` diff --git a/contrib/docker-compose/configure.sh b/contrib/docker-compose/configure.sh new file mode 100755 index 000000000..513d23b27 --- /dev/null +++ b/contrib/docker-compose/configure.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e +cd $(dirname "$0") + +if [[ ! -f .env || "${1}" == "--force" ]] ; then + cp example.env .env +fi + +set -o allexport +source .env +set +o allexport + +if [[ ! -d ./volumes/ingress/certs || "${1}" == "--force" ]] ; then + mkdir -p ../../certs || true + mkdir -p ./volumes/ingress/certs || true + CAROOT=../../certs mkcert -install \ + -cert-file ./volumes/ingress/certs/tls.crt \ + -key-file ./volumes/ingress/certs/tls.key \ + ${APIPROXY_WEB_DOMAIN} \ + ${APIPROXY_AUTH_DOMAIN} \ + ${APIPROXY_API_DOMAIN} +fi + +cat > .env-keys < ./volumes/envoy/config/envoy.yaml + +echo "Done.... now run:" +echo +echo " docker-compose up -d" +echo diff --git a/contrib/docker-compose/docker-compose.yml b/contrib/docker-compose/docker-compose.yml new file mode 100644 index 000000000..620bf1af6 --- /dev/null +++ b/contrib/docker-compose/docker-compose.yml @@ -0,0 +1,197 @@ +version: "3.7" +name: nexodus +networks: + nexodus: +volumes: + db: + +services: + db: + container_name: nexodus_db + image: postgres:16-alpine + networks: + - nexodus + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_DB}" ] + interval: 10s + timeout: 5s + retries: 6 + volumes: + - type: volume + source: db + target: /var/lib/postgresql/data + + ingress: + restart: unless-stopped + container_name: nexodus_ingress + networks: + - nexodus + depends_on: + - apiserver + - frontend + - auth + image: envoyproxy/envoy:v1.27.0 + volumes: + - type: bind + source: ./volumes/ingress/certs + target: /configs/envoy-certs/nexodus-cert + read_only: true + - type: bind + source: ./volumes/envoy/config + target: /configs/envoy + read_only: true + - type: bind + source: ./volumes/envoy/sockets + target: /sockets + command: + - envoy + - --config-path + - /configs/envoy/envoy.yaml + - --component-log-level + - ${ENVOY_COMP_LOG_LEVEL} + ports: + - "443:8443" + + frontend: + restart: unless-stopped + container_name: nexodus_frontend + networks: + - nexodus + build: + context: ../.. + dockerfile: Containerfile.frontend + + apiserver: + restart: unless-stopped + container_name: nexodus_apiserver + networks: + - nexodus + depends_on: + - db + # image: nexodus/nexodus:latest + build: + context: ../.. + dockerfile: Containerfile.apiserver + args: + ARG NEXODUS_PPROF: ${NEXODUS_PPROF} + ARG NEXODUS_RACE_DETECTOR: ${NEXODUS_RACE_DETECTOR} + env_file: + - .env-keys + environment: + NEXAPI_LISTEN: 0.0.0.0:8080 + NEXAPI_DEBUG: ${NEXAPI_DEBUG} + NEXAPI_DB_HOST: db + NEXAPI_DB_PORT: 5432 + NEXAPI_DB_NAME: ${POSTGRES_DB} + NEXAPI_DB_USER: ${POSTGRES_USER} + NEXAPI_DB_PASSWORD: ${POSTGRES_PASSWORD} + NEXAPI_DB_SSLMODE: ${POSTGRES_SSL_MODE} + NEXAPI_IPAM_URL: ${NEXAPI_IPAM_URL} + NEXAPI_OIDC_URL: ${NEXAPI_OIDC_URL} + NEXAPI_OIDC_BACKCHANNEL: ${NEXAPI_OIDC_BACKCHANNEL} + NEXAPI_INSECURE_TLS: ${NEXAPI_INSECURE_TLS} + NEXAPI_OIDC_CLIENT_ID_WEB: ${WEB_CLIENT_ID} + NEXAPI_OIDC_CLIENT_SECRET_WEB: ${WEB_CLIENT_SECRET} + NEXAPI_OIDC_CLIENT_ID_CLI: ${CLI_CLIENT_ID} + NEXAPI_TRACE_ENDPOINT_OTLP: ${NEXAPI_TRACE_ENDPOINT_OTLP} + NEXAPI_TRACE_INSECURE: ${NEXAPI_TRACE_INSECURE} + NEXAPI_FFLAG_SECURITY_GROUPS: ${NEXAPI_FFLAG_SECURITY_GROUPS} + NEXAPI_ORIGINS: ${NEXAPI_ORIGINS} + NEXAPI_DOMAIN: ${NEXAPI_DOMAIN} + NEXAPI_SCOPES: ${NEXAPI_SCOPES} + NEXAPI_REDIS_SERVER: ${NEXAPI_REDIS_SERVER} + NEXAPI_REDIS_DB: ${NEXAPI_REDIS_DB} + NEXAPI_ENVIRONMENT: ${NEXAPI_ENVIRONMENT} + NEXAPI_FETCH_MGR: ${NEXAPI_FETCH_MGR} + NEXAPI_FETCH_MGR_TIMEOUT: ${NEXAPI_FETCH_MGR_TIMEOUT} + NEXAPI_DEVICE_CACHE_SIZE: ${NEXAPI_DEVICE_CACHE_SIZE} + NEXAPI_URL: ${NEXAPI_URL} + NEXAPI_SMTP_FROM: ${NEXAPI_SMTP_FROM} + NEXAPI_SMTP_HOST_PORT: ${NEXAPI_SMTP_HOST_PORT} + NEXAPI_SMTP_PASSWORD: ${NEXAPI_SMTP_PASSWORD} + NEXAPI_SMTP_TLS: ${NEXAPI_SMTP_TLS} + NEXAPI_SMTP_USER: ${NEXAPI_SMTP_USER} + NEXAPI_FFLAG_DEVICES: ${NEXAPI_FFLAG_DEVICES} + NEXAPI_FFLAG_SITES: ${NEXAPI_FFLAG_SITES} + healthcheck: + test: curl --fail http://localhost:8080/private/ready || exit 1 + interval: 5s + timeout: 10s + retries: 6 + volumes: + - type: bind + source: ./volumes/apiserver/sockets + target: /var/lib/apiserver + ports: + - "2345:2345" + # comment the following line to disable remote debugging of the apiserver + command: /dlv --continue --listen=:2345 --api-version=2 --only-same-user=false --headless --accept-multiclient exec /apiserver + + auth: + restart: unless-stopped + container_name: nexodus_auth + networks: + - nexodus + depends_on: + - db + image: quay.io/keycloak/keycloak:21.0.2 + volumes: + - type: bind + source: ../../deploy/nexodus/base/auth/files/nexodus.json + target: /opt/keycloak/data/import/nexodus.json + environment: + KC_DB: "postgres" + KC_DB_URL_HOST: db + KC_DB_URL_PORT: 5432 + KC_DB_URL_DATABASE: ${POSTGRES_DB} + KC_DB_USERNAME: ${POSTGRES_USER} + KC_DB_PASSWORD: ${POSTGRES_PASSWORD} + KC_HOSTNAME: ${KC_HOSTNAME} + KC_PROXY: "edge" + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: "8080" + KEYCLOAK_ADMIN: "admin" + KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + WEB_CLIENT_ID: ${WEB_CLIENT_ID} + WEB_CLIENT_SECRET: ${WEB_CLIENT_SECRET} + CLI_CLIENT_ID: ${CLI_CLIENT_ID} + FRONTEND_URL: ${FRONTEND_URL} + REDIRECT_URL: ${REDIRECT_URL} + GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID} + GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET} + command: [ "start", "--import-realm", "--features=preview", "--features=declarative-user-profile" ] + + ipam: + restart: unless-stopped + container_name: nexodus_ipam + networks: + - nexodus + depends_on: + - db + build: + context: ../.. + dockerfile: Containerfile.ipam + environment: + GOIPAM_PG_HOST: db + GOIPAM_PG_PORT: 5432 + GOIPAM_PG_DBNAME: ${POSTGRES_DB} + GOIPAM_PG_USER: ${POSTGRES_USER} + GOIPAM_PG_PASSWORD: ${POSTGRES_PASSWORD} + GOIPAM_PG_SSLMODE: ${POSTGRES_SSL_MODE} + command: + - --grpc-server-endpoint=0.0.0.0:9090 + - postgres + + redis: + restart: unless-stopped + container_name: nexodus_redis + networks: + - nexodus + image: redis:6.0 + command: [ "--maxmemory", "200mb", "--maxmemory-policy", "allkeys-lru", "--save", "" ] + volumes: [] diff --git a/contrib/docker-compose/example.env b/contrib/docker-compose/example.env new file mode 100644 index 000000000..9b25dbdd5 --- /dev/null +++ b/contrib/docker-compose/example.env @@ -0,0 +1,78 @@ +# +# BUILD SETTINGS +# +NEXODUS_PPROF= +NEXODUS_RACE_DETECTOR= + +# +# DB SETTINGS +# +POSTGRES_USER=nexodus +POSTGRES_DB=nexodus +POSTGRES_PASSWORD=nexodus +POSTGRES_SSL_MODE=disable + +# +# ingress settings +# +APIPROXY_OIDC_URL=https://auth.try.nexodus.127.0.0.1.nip.io/realms/nexodus +APIPROXY_OIDC_BACKCHANNEL=https://auth:8080/realms/nexodus +APIPROXY_API_DOMAIN=api.try.nexodus.127.0.0.1.nip.io +APIPROXY_API_URL=https://api.try.nexodus.127.0.0.1.nip.io +APIPROXY_WEB_DOMAIN=try.nexodus.127.0.0.1.nip.io +APIPROXY_AUTH_DOMAIN=auth.try.nexodus.127.0.0.1.nip.io +APIPROXY_WEB_ORIGINS=https://try.nexodus.127.0.0.1.nip.io +ENVOY_COMP_LOG_LEVEL=upstream:info,http:debug,router:debug,jwt:debug + + +# +# auth settings +# +KC_HOSTNAME=auth.try.nexodus.127.0.0.1.nip.io +FRONTEND_URL=https://try.nexodus.127.0.0.1.nip.io +REDIRECT_URL=https://api.try.nexodus.127.0.0.1.nip.io/web +KEYCLOAK_ADMIN_PASSWORD=floofykittens +WEB_CLIENT_ID=nexodus-web +WEB_CLIENT_SECRET=dhEN2dsqyUg5qmaDAdqi4CmH +CLI_CLIENT_ID=nexodus-cli +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +# +# apiserver settings +# +NEXAPI_DEBUG=1 +NEXAPI_ENVIRONMENT=development +NEXAPI_DOMAIN=api.try.nexodus.127.0.0.1.nip.io +NEXAPI_URL=https://api.try.nexodus.127.0.0.1.nip.io +NEXAPI_ORIGINS=https://try.nexodus.127.0.0.1.nip.io + +NEXAPI_IPAM_URL=http://ipam:9090 + +NEXAPI_OIDC_URL=https://auth.try.nexodus.127.0.0.1.nip.io/realms/nexodus +NEXAPI_OIDC_BACKCHANNEL=http://auth:8080/realms/nexodus +NEXAPI_INSECURE_TLS=1 +NEXAPI_TRACE_ENDPOINT_OTLP="tempo.nexodus-monitoring.svc:431" +NEXAPI_TRACE_INSECURE="1" +NEXAPI_SCOPES=read:organizations,write:organizations,read:users,write:users,read:devices,write:devices + +NEXAPI_REDIS_SERVER=redis:6379 +NEXAPI_REDIS_DB=1 + +NEXAPI_FFLAG_DEVICES=true +NEXAPI_FFLAG_SECURITY_GROUPS=true +NEXAPI_FFLAG_SITES=true + +NEXAPI_FETCH_MGR=redis +NEXAPI_FETCH_MGR_TIMEOUT=2s +NEXAPI_DEVICE_CACHE_SIZE=500 + +NEXAPI_SMTP_HOST_PORT= +NEXAPI_SMTP_TLS= +NEXAPI_SMTP_USER= +NEXAPI_SMTP_PASSWORD= +NEXAPI_SMTP_FROM= + +NEXAPI_CA_CERT= +NEXAPI_TLS_KEY= +NEXAPI_CA_KEY=