Add nginx based reverse proxy (#894)

* Start fleshing out simple docker nginx reverse proxy

* refactor: forward the X-Real-IP header

* Add support for clickhouse proxy inside of reverse-proxy config

---------

Co-authored-by: DecFox <33030671+DecFox@users.noreply.github.com>

ooniapi/services/reverseproxy/Dockerfile

@@ -0,0 +1,6 @@
+FROM nginx:1.27.1
+ENV TARGET_URL="https://backend-hel.ooni.org"
+ENV CLICKHOUSE_STREAM_TARGET="clickhouse1.prod.ooni.io:9000"
+
+COPY templates/backend-proxy.conf.template /etc/nginx/templates/
+COPY templates/clickhouse-proxy.conf.stream-template /etc/nginx/templates/

ooniapi/services/reverseproxy/Makefile

@@ -0,0 +1,62 @@
+SERVICE_NAME ?= reverseproxy
+PKG_VERSION = 1.0.0
+
+ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME)
+IMAGE_NAME ?= ooni/api-$(SERVICE_NAME)
+DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))")
+GIT_FULL_SHA ?= $(shell git rev-parse HEAD)
+SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8)
+
+BUILD_LABEL := $(DATE)-$(SHORT_SHA)
+VERSION_LABEL = v$(PKG_VERSION)
+ENV_LABEL ?= latest
+
+print-labels:
+	echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}"
+	echo "PKG_VERSION=${PKG_VERSION}"
+	echo "BUILD_LABEL=${BUILD_LABEL}"
+	echo "VERSION_LABEL=${VERSION_LABEL}"
+	echo "ENV_LABEL=${ENV_LABEL}"
+
+init:
+	echo "noop"
+
+docker-build:
+	docker build \
+		--build-arg BUILD_LABEL=${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${VERSION_LABEL} \
+		-t ${IMAGE_NAME}:${ENV_LABEL} \
+		.
+	echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})"
+
+docker-push:
+	docker push ${IMAGE_NAME}:${BUILD_LABEL}
+	docker push ${IMAGE_NAME}:${VERSION_LABEL}
+	docker push ${IMAGE_NAME}:${ENV_LABEL}
+
+docker-smoketest:
+	./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL}
+
+imagedefinitions.json:
+	echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json
+
+test:
+	echo "noop"
+
+test-cov:
+	echo "noop"
+
+build:
+	docker build .
+
+clean:
+	hatch clean
+	rm -f imagedefinitions.json
+	rm -rf build dist *eggs *.egg-info
+	rm -rf .venv
+
+run:
+	./run.sh
+
+.PHONY: init test build clean docker

ooniapi/services/reverseproxy/buildspec.yml

@@ -0,0 +1,29 @@
+version: 0.2
+env:
+  variables:
+    OONI_CODE_PATH: ooniapi/services/reverseproxy
+    DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+
+  pre_build:
+    commands:
+      - echo "Logging in to dockerhub"
+      - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text)
+      - echo $DOCKER_SECRET | docker login --username ooni --password-stdin
+
+  build:
+    commands:
+      - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION}
+      - cd $OONI_CODE_PATH
+      - make docker-build
+      - make docker-smoketest
+      - make docker-push
+      - make imagedefinitions.json
+      - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json
+
+artifacts:
+  files: imagedefinitions.json

ooniapi/services/reverseproxy/run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -ex
+
+docker run -p 8080:80 --rm -it $(docker build -q .)

ooniapi/services/reverseproxy/scripts/docker-smoketest.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -ex
+
+if [ $# -eq 0 ]; then
+  echo "Error: No Docker image name provided."
+  echo "Usage: $0 [IMAGE_NAME]"
+  exit 1
+fi
+
+IMAGE=$1
+CONTAINER_NAME=ooniapi-smoketest-$RANDOM
+PORT=$((RANDOM % 10001 + 30000))
+
+cleanup() {
+    echo "cleaning up"
+    docker logs $CONTAINER_NAME
+    docker stop $CONTAINER_NAME >/dev/null 2>&1
+    docker rm $CONTAINER_NAME >/dev/null 2>&1
+}
+
+echo "[+] Running smoketest of ${IMAGE}"
+docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE}
+
+trap cleanup INT TERM EXIT
+
+sleep 2
+response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health)
+if [ "${response}" -eq 200 ]; then
+  echo "Smoke test passed: Received 200 OK from /health endpoint."
+else
+  echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response"
+  exit 1
+fi

ooniapi/services/reverseproxy/templates/backend-proxy.conf.template

@@ -0,0 +1,30 @@
+gzip on;
+gzip_proxied any;
+gzip_types text/plain application/json;
+gzip_min_length 1000;
+
+server {
+    listen 8080;
+    location /stub_status {
+        stub_status   on;
+    }
+}
+
+server {
+    listen 80;
+
+    location / {
+        if ($request_method !~ ^(GET|POST|HEAD|OPTIONS|PUT|DELETE)$) {
+            return 405;
+        }
+
+        proxy_pass ${TARGET_URL};
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'upgrade';
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $http_x_forwarded_for;
+        proxy_cache_bypass $http_upgrade;
+    }
+}

ooniapi/services/reverseproxy/templates/clickhouse-proxy.conf.stream-template

@@ -0,0 +1,8 @@
+upstream clickhouse_backend {
+    server ${CLICKHOUSE_STREAM_TARGET};
+}
+
+server {
+    listen 9000;
+    proxy_pass clickhouse_backend; 
+}