Service ooniauth v1 (#823)

Fixes: #821

* Add docs on creating service

* Add more docs about ooni services setup

* Improvements to docs

* Implement ooniauth service

* Reach feature parity with legacyapi

* Setup Dockerfile and smoketest

* Implement tests for the authenatication endpoint

* Add CI for ooniauth

* Add buildspec

* Fix name of tests

* hatch clean shouldn't be in clean

* Implement more health checks add simple smoke test

* Add more tests

* Fix smoketest

* Reach 99% code coverage

The only missing lines are those related to setting up clickhouse and
boto. We should eventually test these too inside of integration tests.

* Be a bit more lenient in code coverage checks 95% target is reasonable

* Long docstrings are redundant

* Set patch target to 95% too

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 95
+    patch:
+      default:
+        target: 95

.github/workflows/test_ooniapi_ooniauth.yml

@@ -0,0 +1,25 @@
+name: test ooniapi/ooniauth
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Run all tests
+        run: make test-cov
+        working-directory: ./ooniapi/services/ooniauth/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ooniauth
+          working-directory: ./ooniapi/services/ooniauth/

.github/workflows/test_ooniapi_oonirun.yml

@@ -21,4 +21,5 @@ jobs:
       - name: Upload coverage to codecov
         uses: codecov/codecov-action@v3
         with:
+          flags: oonirun
           working-directory: ./ooniapi/services/oonirun/

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-    "python.defaultInterpreterPath": "${workspaceFolder}/ooniapi/services/oonirun/.venv",
+    "python.defaultInterpreterPath": "${workspaceFolder}/ooniapi/services/ooniauth/.venv",
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true
 }

ooniapi/common/src/common/config.py

@@ -1,5 +1,3 @@
-import statsd
-
 from typing import List
 from pydantic_settings import BaseSettings
 
@@ -8,13 +6,20 @@ class Settings(BaseSettings):
     app_name: str = "OONI Data API"
     base_url: str = "https://api.ooni.io"
     clickhouse_url: str = "clickhouse://localhost"
-    # In production you want to set this to: postgresql://user:password@postgresserver/db
-    postgresql_url: str = "sqlite:///./testdb.sqlite3"
+    postgresql_url: str = "postgresql://oonidb:oonidb@localhost/oonidb"
     log_level: str = "info"
     s3_bucket_name: str = "oonidata-eufra"
     other_collectors: List[str] = []
     statsd_host: str = "localhost"
     statsd_port: int = 8125
     statsd_prefix: str = "ooniapi"
     jwt_encryption_key: str = "CHANGEME"
+    account_id_hashing_key: str = "CHANGEME"
     prometheus_metrics_password: str = "CHANGEME"
+    session_expiry_days: int = 10
+    login_expiry_days: int = 10
+
+    aws_region: str = ""
+    aws_access_key_id: str = ""
+    aws_secret_access_key: str = ""
+    email_source_address: str = "contact+dev@ooni.io"

ooniapi/common/src/common/routers.py

@@ -0,0 +1,14 @@
+from datetime import date, datetime
+from pydantic import BaseModel as PydandicBaseModel
+
+
+ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ"
+ISO_FORMAT_DATE = "%Y-%m-%d"
+
+
+class BaseModel(PydandicBaseModel):
+    class Config:
+        json_encoders = {
+            datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME),
+            date: lambda v: v.strftime(ISO_FORMAT_DATE),
+        }

ooniapi/services/README.md

@@ -1,7 +1,3 @@
----
-title: "OONI Services"
----
-
 OONI API components are broken up into smaller pieces that can be more easily
 deployed and managed without worrying too much about the blast radius caused by
 the deployment of a larger component.
@@ -402,6 +398,47 @@ CMD ["uvicorn", "ooniservicename.main:app", "--host", "0.0.0.0", "--port", "80"]
 EXPOSE 80
 ```
 
+It's recommended you also implement a smoke test for the built docker image.
+
+Here is a sample:
+
+```bash
+#!/bin/bash
+
+set -ex
+
+if [ $# -eq 0 ]; then
+  echo "Error: No Docker image name provided."
+  echo "Usage: $0 [IMAGE_NAME]"
+  exit 1
+fi
+
+IMAGE=$1
+CONTAINER_NAME=ooniapi-smoketest-$RANDOM
+PORT=$((RANDOM % 10001 + 30000))
+
+cleanup() {
+    echo "cleaning up"
+    docker logs $CONTAINER_NAME
+    docker stop $CONTAINER_NAME >/dev/null 2>&1
+    docker rm $CONTAINER_NAME >/dev/null 2>&1
+}
+
+echo "[+] Running smoketest of ${IMAGE}"
+docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE}
+
+trap cleanup INT TERM EXIT
+
+sleep 2
+response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health)
+if [ "${response}" -eq 200 ]; then
+  echo "Smoke test passed: Received 200 OK from /health endpoint."
+else
+  echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response"
+  exit 1
+fi
+```
+
 #### Build spec
 
 The service must implement a `buildspec.yml` file that specifies how to

ooniapi/services/ooniauth/.gitignore

@@ -0,0 +1 @@
+dist/

ooniapi/services/ooniauth/Dockerfile

@@ -0,0 +1,33 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=dev
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/ooniauth/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+#COPY --from=builder /build/alembic/ /app/alembic/
+#COPY --from=builder /build/alembic.ini /app/
+#RUN rm -rf /app/alembic/__pycache__
+
+CMD ["uvicorn", "ooniauth.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/ooniauth/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ooniapi/services/ooniauth/Makefile

@@ -0,0 +1,63 @@
+SERVICE_NAME ?= ooniauth
+
+ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME)
+IMAGE_NAME ?= ooni/api-$(SERVICE_NAME)
+DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))")
+GIT_FULL_SHA ?= $(shell git rev-parse HEAD)
+SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8)
+PKG_VERSION := $(shell hatch version)
+
+BUILD_LABEL := $(DATE)-$(SHORT_SHA)
+VERSION_LABEL = v$(PKG_VERSION)
+ENV_LABEL ?= latest
+
+print-labels:
+	echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}"
+	echo "PKG_VERSION=${PKG_VERSION}"
+	echo "BUILD_LABEL=${BUILD_LABEL}"
+	echo "VERSION_LABEL=${VERSION_LABEL}"
+	echo "ENV_LABEL=${ENV_LABEL}"
+
+init:
+	hatch env create
+
+docker-build: clean
+	# We need to use tar -czh to resolve the common dir symlink
+	tar -czh . | docker build \
+		--build-arg BUILD_LABEL=${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${VERSION_LABEL} \
+		-t ${IMAGE_NAME}:${ENV_LABEL} \
+		-
+	echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})"
+
+docker-push:
+	# We need to use tar -czh to resolve the common dir symlink
+	docker push ${IMAGE_NAME}:${BUILD_LABEL}
+	docker push ${IMAGE_NAME}:${VERSION_LABEL}
+	docker push ${IMAGE_NAME}:${ENV_LABEL}
+
+docker-smoketest:
+	./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL}
+
+imagedefinitions.json:
+	echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json
+
+test:
+	hatch run test
+
+test-cov:
+	hatch run test-cov
+
+build:
+	hatch build
+
+clean:
+	rm -f imagedefinitions.json
+	rm -rf build dist *eggs *.egg-info
+	rm -rf .venv
+
+run:
+	hatch run uvicorn $(SERVICE_NAME).main:app
+
+.PHONY: init test build clean docker print-labels

ooniapi/services/ooniauth/README.md

@@ -0,0 +1,21 @@
+# ooniauth
+
+[![PyPI - Version](https://img.shields.io/pypi/v/ooniauth.svg)](https://pypi.org/project/ooniauth)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/ooniauth.svg)](https://pypi.org/project/ooniauth)
+
+-----
+
+**Table of Contents**
+
+- [Installation](#installation)
+- [License](#license)
+
+## Installation
+
+```console
+pip install ooniauth
+```
+
+## License
+
+`ooniauth` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license.

ooniapi/services/ooniauth/buildspec.yml

@@ -0,0 +1,29 @@
+version: 0.2
+env:
+  variables:
+    OONI_CODE_PATH: ooniapi/services/ooniauth
+    DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+
+  pre_build:
+    commands:
+      - echo "Logging in to dockerhub"
+      - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text)
+      - echo $DOCKER_SECRET | docker login --username ooni --password-stdin
+
+  build:
+    commands:
+      - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION}
+      - cd $OONI_CODE_PATH
+      - make docker-build
+      - make docker-smoketest
+      - make docker-push
+      - make imagedefinitions.json
+      - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json
+
+artifacts:
+  files: imagedefinitions.json