From 0c9bdd94886b014d940158db81d4dccfa2ccb143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 15 Mar 2024 07:36:27 +0100 Subject: [PATCH] Service ooniauth v1 (#823) Fixes: https://github.com/ooni/backend/issues/821 * Add docs on creating service * Add more docs about ooni services setup * Improvements to docs * Implement ooniauth service * Reach feature parity with legacyapi * Setup Dockerfile and smoketest * Implement tests for the authenatication endpoint * Add CI for ooniauth * Add buildspec * Fix name of tests * hatch clean shouldn't be in clean * Implement more health checks add simple smoke test * Add more tests * Fix smoketest * Reach 99% code coverage The only missing lines are those related to setting up clickhouse and boto. We should eventually test these too inside of integration tests. * Be a bit more lenient in code coverage checks 95% target is reasonable * Long docstrings are redundant * Set patch target to 95% too --- .codecov.yml | 8 + .github/workflows/test_ooniapi_ooniauth.yml | 25 ++ .github/workflows/test_ooniapi_oonirun.yml | 1 + .vscode/settings.json | 2 +- ooniapi/common/src/common/config.py | 13 +- ooniapi/common/src/common/routers.py | 14 ++ ooniapi/services/README.md | 45 +++- ooniapi/services/ooniauth/.gitignore | 1 + ooniapi/services/ooniauth/Dockerfile | 33 +++ ooniapi/services/ooniauth/LICENSE.txt | 26 ++ ooniapi/services/ooniauth/Makefile | 63 +++++ ooniapi/services/ooniauth/README.md | 21 ++ ooniapi/services/ooniauth/buildspec.yml | 29 +++ ooniapi/services/ooniauth/pyproject.toml | 100 ++++++++ .../ooniauth/scripts/docker-smoketest.sh | 42 ++++ .../ooniauth/src/ooniauth/__about__.py | 1 + .../ooniauth/src/ooniauth/__init__.py | 3 + ooniapi/services/ooniauth/src/ooniauth/common | 1 + .../ooniauth/src/ooniauth/dependencies.py | 24 ++ .../services/ooniauth/src/ooniauth/main.py | 101 ++++++++ .../ooniauth/src/ooniauth/routers/v1.py | 227 ++++++++++++++++++ .../services/ooniauth/src/ooniauth/utils.py | 103 ++++++++ ooniapi/services/ooniauth/tests/__init__.py | 3 + ooniapi/services/ooniauth/tests/conftest.py | 120 +++++++++ .../services/ooniauth/tests/run_live_test.py | 14 ++ ooniapi/services/ooniauth/tests/test_auth.py | 196 +++++++++++++++ ooniapi/services/ooniauth/tests/test_main.py | 55 +++++ ooniapi/services/oonirun/src/oonirun/main.py | 4 +- .../oonirun/src/oonirun/routers/oonirun.py | 13 +- 29 files changed, 1266 insertions(+), 22 deletions(-) create mode 100644 .codecov.yml create mode 100644 .github/workflows/test_ooniapi_ooniauth.yml create mode 100644 ooniapi/common/src/common/routers.py create mode 100644 ooniapi/services/ooniauth/.gitignore create mode 100644 ooniapi/services/ooniauth/Dockerfile create mode 100644 ooniapi/services/ooniauth/LICENSE.txt create mode 100644 ooniapi/services/ooniauth/Makefile create mode 100644 ooniapi/services/ooniauth/README.md create mode 100644 ooniapi/services/ooniauth/buildspec.yml create mode 100644 ooniapi/services/ooniauth/pyproject.toml create mode 100755 ooniapi/services/ooniauth/scripts/docker-smoketest.sh create mode 100644 ooniapi/services/ooniauth/src/ooniauth/__about__.py create mode 100644 ooniapi/services/ooniauth/src/ooniauth/__init__.py create mode 120000 ooniapi/services/ooniauth/src/ooniauth/common create mode 100644 ooniapi/services/ooniauth/src/ooniauth/dependencies.py create mode 100644 ooniapi/services/ooniauth/src/ooniauth/main.py create mode 100644 ooniapi/services/ooniauth/src/ooniauth/routers/v1.py create mode 100644 ooniapi/services/ooniauth/src/ooniauth/utils.py create mode 100644 ooniapi/services/ooniauth/tests/__init__.py create mode 100644 ooniapi/services/ooniauth/tests/conftest.py create mode 100644 ooniapi/services/ooniauth/tests/run_live_test.py create mode 100644 ooniapi/services/ooniauth/tests/test_auth.py create mode 100644 ooniapi/services/ooniauth/tests/test_main.py diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..42be417b --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + target: 95 + patch: + default: + target: 95 diff --git a/.github/workflows/test_ooniapi_ooniauth.yml b/.github/workflows/test_ooniapi_ooniauth.yml new file mode 100644 index 00000000..139a0429 --- /dev/null +++ b/.github/workflows/test_ooniapi_ooniauth.yml @@ -0,0 +1,25 @@ +name: test ooniapi/ooniauth +on: push +jobs: + run_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install hatch + run: pip install hatch + + - name: Run all tests + run: make test-cov + working-directory: ./ooniapi/services/ooniauth/ + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v3 + with: + flags: ooniauth + working-directory: ./ooniapi/services/ooniauth/ diff --git a/.github/workflows/test_ooniapi_oonirun.yml b/.github/workflows/test_ooniapi_oonirun.yml index 92f73f77..86e4dfd6 100644 --- a/.github/workflows/test_ooniapi_oonirun.yml +++ b/.github/workflows/test_ooniapi_oonirun.yml @@ -21,4 +21,5 @@ jobs: - name: Upload coverage to codecov uses: codecov/codecov-action@v3 with: + flags: oonirun working-directory: ./ooniapi/services/oonirun/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 977d268c..b1809e76 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "python.defaultInterpreterPath": "${workspaceFolder}/ooniapi/services/oonirun/.venv", + "python.defaultInterpreterPath": "${workspaceFolder}/ooniapi/services/ooniauth/.venv", "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } diff --git a/ooniapi/common/src/common/config.py b/ooniapi/common/src/common/config.py index e275444c..9e4a6767 100644 --- a/ooniapi/common/src/common/config.py +++ b/ooniapi/common/src/common/config.py @@ -1,5 +1,3 @@ -import statsd - from typing import List from pydantic_settings import BaseSettings @@ -8,8 +6,7 @@ class Settings(BaseSettings): app_name: str = "OONI Data API" base_url: str = "https://api.ooni.io" clickhouse_url: str = "clickhouse://localhost" - # In production you want to set this to: postgresql://user:password@postgresserver/db - postgresql_url: str = "sqlite:///./testdb.sqlite3" + postgresql_url: str = "postgresql://oonidb:oonidb@localhost/oonidb" log_level: str = "info" s3_bucket_name: str = "oonidata-eufra" other_collectors: List[str] = [] @@ -17,4 +14,12 @@ class Settings(BaseSettings): statsd_port: int = 8125 statsd_prefix: str = "ooniapi" jwt_encryption_key: str = "CHANGEME" + account_id_hashing_key: str = "CHANGEME" prometheus_metrics_password: str = "CHANGEME" + session_expiry_days: int = 10 + login_expiry_days: int = 10 + + aws_region: str = "" + aws_access_key_id: str = "" + aws_secret_access_key: str = "" + email_source_address: str = "contact+dev@ooni.io" diff --git a/ooniapi/common/src/common/routers.py b/ooniapi/common/src/common/routers.py new file mode 100644 index 00000000..80868140 --- /dev/null +++ b/ooniapi/common/src/common/routers.py @@ -0,0 +1,14 @@ +from datetime import date, datetime +from pydantic import BaseModel as PydandicBaseModel + + +ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ" +ISO_FORMAT_DATE = "%Y-%m-%d" + + +class BaseModel(PydandicBaseModel): + class Config: + json_encoders = { + datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME), + date: lambda v: v.strftime(ISO_FORMAT_DATE), + } diff --git a/ooniapi/services/README.md b/ooniapi/services/README.md index d5812cb2..2a11aeb3 100644 --- a/ooniapi/services/README.md +++ b/ooniapi/services/README.md @@ -1,7 +1,3 @@ ---- -title: "OONI Services" ---- - OONI API components are broken up into smaller pieces that can be more easily deployed and managed without worrying too much about the blast radius caused by the deployment of a larger component. @@ -402,6 +398,47 @@ CMD ["uvicorn", "ooniservicename.main:app", "--host", "0.0.0.0", "--port", "80"] EXPOSE 80 ``` +It's recommended you also implement a smoke test for the built docker image. + +Here is a sample: + +```bash +#!/bin/bash + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi +``` + #### Build spec The service must implement a `buildspec.yml` file that specifies how to diff --git a/ooniapi/services/ooniauth/.gitignore b/ooniapi/services/ooniauth/.gitignore new file mode 100644 index 00000000..849ddff3 --- /dev/null +++ b/ooniapi/services/ooniauth/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/ooniapi/services/ooniauth/Dockerfile b/ooniapi/services/ooniauth/Dockerfile new file mode 100644 index 00000000..68416616 --- /dev/null +++ b/ooniapi/services/ooniauth/Dockerfile @@ -0,0 +1,33 @@ +# Python builder +FROM python:3.11-bookworm as builder +ARG BUILD_LABEL=dev + +WORKDIR /build + +RUN python -m pip install hatch + +COPY . /build + +# When you build stuff on macOS you end up with ._ files +# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them +RUN find /build -type f -name '._*' -delete + +RUN echo "$BUILD_LABEL" > /build/src/ooniauth/BUILD_LABEL + +RUN hatch build + +### Actual image running on the host +FROM python:3.11-bookworm as runner + +WORKDIR /app + +COPY --from=builder /build/README.md /app/ +COPY --from=builder /build/dist/*.whl /app/ +RUN pip install /app/*whl && rm /app/*whl + +#COPY --from=builder /build/alembic/ /app/alembic/ +#COPY --from=builder /build/alembic.ini /app/ +#RUN rm -rf /app/alembic/__pycache__ + +CMD ["uvicorn", "ooniauth.main:app", "--host", "0.0.0.0", "--port", "80"] +EXPOSE 80 diff --git a/ooniapi/services/ooniauth/LICENSE.txt b/ooniapi/services/ooniauth/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/ooniapi/services/ooniauth/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ooniapi/services/ooniauth/Makefile b/ooniapi/services/ooniauth/Makefile new file mode 100644 index 00000000..7d840bfe --- /dev/null +++ b/ooniapi/services/ooniauth/Makefile @@ -0,0 +1,63 @@ +SERVICE_NAME ?= ooniauth + +ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME) +IMAGE_NAME ?= ooni/api-$(SERVICE_NAME) +DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))") +GIT_FULL_SHA ?= $(shell git rev-parse HEAD) +SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8) +PKG_VERSION := $(shell hatch version) + +BUILD_LABEL := $(DATE)-$(SHORT_SHA) +VERSION_LABEL = v$(PKG_VERSION) +ENV_LABEL ?= latest + +print-labels: + echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}" + echo "PKG_VERSION=${PKG_VERSION}" + echo "BUILD_LABEL=${BUILD_LABEL}" + echo "VERSION_LABEL=${VERSION_LABEL}" + echo "ENV_LABEL=${ENV_LABEL}" + +init: + hatch env create + +docker-build: clean + # We need to use tar -czh to resolve the common dir symlink + tar -czh . | docker build \ + --build-arg BUILD_LABEL=${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${BUILD_LABEL} \ + -t ${IMAGE_NAME}:${VERSION_LABEL} \ + -t ${IMAGE_NAME}:${ENV_LABEL} \ + - + echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})" + +docker-push: + # We need to use tar -czh to resolve the common dir symlink + docker push ${IMAGE_NAME}:${BUILD_LABEL} + docker push ${IMAGE_NAME}:${VERSION_LABEL} + docker push ${IMAGE_NAME}:${ENV_LABEL} + +docker-smoketest: + ./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL} + +imagedefinitions.json: + echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json + +test: + hatch run test + +test-cov: + hatch run test-cov + +build: + hatch build + +clean: + rm -f imagedefinitions.json + rm -rf build dist *eggs *.egg-info + rm -rf .venv + +run: + hatch run uvicorn $(SERVICE_NAME).main:app + +.PHONY: init test build clean docker print-labels diff --git a/ooniapi/services/ooniauth/README.md b/ooniapi/services/ooniauth/README.md new file mode 100644 index 00000000..889be032 --- /dev/null +++ b/ooniapi/services/ooniauth/README.md @@ -0,0 +1,21 @@ +# ooniauth + +[![PyPI - Version](https://img.shields.io/pypi/v/ooniauth.svg)](https://pypi.org/project/ooniauth) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/ooniauth.svg)](https://pypi.org/project/ooniauth) + +----- + +**Table of Contents** + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install ooniauth +``` + +## License + +`ooniauth` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. diff --git a/ooniapi/services/ooniauth/buildspec.yml b/ooniapi/services/ooniauth/buildspec.yml new file mode 100644 index 00000000..c54fbd53 --- /dev/null +++ b/ooniapi/services/ooniauth/buildspec.yml @@ -0,0 +1,29 @@ +version: 0.2 +env: + variables: + OONI_CODE_PATH: ooniapi/services/ooniauth + DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token + +phases: + install: + runtime-versions: + python: 3.11 + + pre_build: + commands: + - echo "Logging in to dockerhub" + - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text) + - echo $DOCKER_SECRET | docker login --username ooni --password-stdin + + build: + commands: + - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION} + - cd $OONI_CODE_PATH + - make docker-build + - make docker-smoketest + - make docker-push + - make imagedefinitions.json + - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json + +artifacts: + files: imagedefinitions.json diff --git a/ooniapi/services/ooniauth/pyproject.toml b/ooniapi/services/ooniauth/pyproject.toml new file mode 100644 index 00000000..8bd64c2a --- /dev/null +++ b/ooniapi/services/ooniauth/pyproject.toml @@ -0,0 +1,100 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "ooniauth" +dynamic = ["version"] +description = '' +readme = "README.md" +requires-python = ">=3.8" +license = "BSD-3-Clause" +authors = [{ name = "OONI", email = "contact@ooni.org" }] +keywords = [] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + "fastapi ~= 0.108.0", + "clickhouse-driver ~= 0.2.6", + "sqlalchemy ~= 2.0.27", + "ujson ~= 5.9.0", + "python-dateutil ~= 2.8.2", + "pydantic-settings ~= 2.1.0", + "uvicorn ~= 0.25.0", + "psycopg2 ~= 2.9.9", + "httpx ~= 0.26.0", + "pyjwt ~= 2.8.0", + "alembic ~= 1.13.1", + "prometheus-fastapi-instrumentator ~= 6.1.0", + "prometheus-client", + "email-validator", + "boto3 ~= 1.34.0", +] + +[project.urls] +Documentation = "https://docs.ooni.org/" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/ooniauth/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/ooniauth"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-postgresql", + "pytest-asyncio", + "freezegun", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = ["mypy>=1.0.0"] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/ooniauth tests}" + +[tool.coverage.run] +source_pkgs = ["ooniauth", "tests"] +branch = true +parallel = true +omit = [ + "src/ooniauth/__about__.py", + "src/ooniauth/common/*", + # Ignored because these should be run manually on deployed instance + "tests/run_*", +] + +[tool.coverage.paths] +ooniauth = ["src/ooniauth", "*/ooniauth/src/ooniauth"] +tests = ["tests", "*/ooniauth/tests"] + +[tool.coverage.report] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] diff --git a/ooniapi/services/ooniauth/scripts/docker-smoketest.sh b/ooniapi/services/ooniauth/scripts/docker-smoketest.sh new file mode 100755 index 00000000..4338cfa2 --- /dev/null +++ b/ooniapi/services/ooniauth/scripts/docker-smoketest.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -ex + +if [ $# -eq 0 ]; then + echo "Error: No Docker image name provided." + echo "Usage: $0 [IMAGE_NAME]" + exit 1 +fi + +IMAGE=$1 +CONTAINER_NAME=ooniapi-smoketest-$RANDOM +PORT=$((RANDOM % 10001 + 30000)) + +cleanup() { + echo "cleaning up" + docker logs $CONTAINER_NAME + docker stop $CONTAINER_NAME >/dev/null 2>&1 + docker rm $CONTAINER_NAME >/dev/null 2>&1 +} + +echo "[+] Running smoketest of ${IMAGE}" +docker run \ + -e AWS_ACCESS_KEY_ID=ITSCHANGED \ + -e AWS_ACCESS_KEY_ID=ITSCHANGED \ + -e PROMETHEUS_METRICS_PASSWORD=ITSCHANGED \ + -e JWT_ENCRYPTION_KEY=ITCHANGED \ + -e ACCOUNT_ID_HASHING_KEY=ITSCHANGED \ + -e AWS_ACCESS_KEY_ID=ITSCHANGED \ + -e AWS_SECRET_ACCESS_KEY=ITSCHANGED \ + -d --name $CONTAINER_NAME -p $PORT:80 ${IMAGE} + +trap cleanup INT TERM EXIT + +sleep 2 +response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:$PORT/health) +if [ "${response}" -eq 200 ]; then + echo "Smoke test passed: Received 200 OK from /health endpoint." +else + echo "Smoke test failed: Did not receive 200 OK from /health endpoint. Received: $response" + exit 1 +fi diff --git a/ooniapi/services/ooniauth/src/ooniauth/__about__.py b/ooniapi/services/ooniauth/src/ooniauth/__about__.py new file mode 100644 index 00000000..3dc1f76b --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/__about__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/ooniapi/services/ooniauth/src/ooniauth/__init__.py b/ooniapi/services/ooniauth/src/ooniauth/__init__.py new file mode 100644 index 00000000..632b5b71 --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2024-present Arturo Filastò +# +# SPDX-License-Identifier: MIT diff --git a/ooniapi/services/ooniauth/src/ooniauth/common b/ooniapi/services/ooniauth/src/ooniauth/common new file mode 120000 index 00000000..3f599f25 --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/common @@ -0,0 +1 @@ +../../../../common/src/common \ No newline at end of file diff --git a/ooniapi/services/ooniauth/src/ooniauth/dependencies.py b/ooniapi/services/ooniauth/src/ooniauth/dependencies.py new file mode 100644 index 00000000..ee4b6ab0 --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/dependencies.py @@ -0,0 +1,24 @@ +from typing import Annotated + +from clickhouse_driver import Client as ClickhouseClient +import boto3 + +from fastapi import Depends + +from .common.dependencies import get_settings +from .common.config import Settings + + +def get_clickhouse_client( + settings: Annotated[Settings, Depends(get_settings)] +) -> ClickhouseClient: + return ClickhouseClient.from_url(settings.clickhouse_url) + + +def get_ses_client(settings: Annotated[Settings, Depends(get_settings)]): + return boto3.client( + "ses", + region_name=settings.aws_region, + aws_access_key_id=settings.aws_access_key_id, + aws_secret_access_key=settings.aws_secret_access_key, + ) diff --git a/ooniapi/services/ooniauth/src/ooniauth/main.py b/ooniapi/services/ooniauth/src/ooniauth/main.py new file mode 100644 index 00000000..a6dd44ed --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/main.py @@ -0,0 +1,101 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware + +from pydantic import BaseModel + +from prometheus_fastapi_instrumentator import Instrumentator + +from .routers import v1 + +from .common.config import Settings +from .common.dependencies import get_settings +from .common.version import get_build_label, get_pkg_version +from .common.metrics import mount_metrics + + +log = logging.getLogger(__name__) + +pkg_name = "ooniauth" + +pkg_version = get_pkg_version(pkg_name) +build_label = get_build_label(pkg_name) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + logging.basicConfig(level=getattr(logging, settings.log_level.upper())) + mount_metrics(app, instrumentor.registry) + yield + + +app = FastAPI(lifespan=lifespan) + +instrumentor = Instrumentator().instrument( + app, metric_namespace="ooniapi", metric_subsystem="ooniauth" +) + +# TODO: temporarily enable all +origins = ["*"] +app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(v1.router, prefix="/api") + + +@app.get("/version") +async def version(): + return { + "version": pkg_version, + "build_label": build_label, + "package_name": pkg_name, + } + + +class HealthStatus(BaseModel): + status: str + errors: list[str] = [] + version: str + build_label: str + + +@app.get("/health") +async def health( + settings: Settings = Depends(get_settings), +): + errors = [] + + if settings.jwt_encryption_key == "CHANGEME": + errors.append("bad_jwt_secret") + + if settings.prometheus_metrics_password == "CHANGEME": + errors.append("bad_prometheus_password") + + if settings.aws_secret_access_key == "" or settings.aws_access_key_id == "": + errors.append("bad_aws_credentials") + + if settings.account_id_hashing_key == "CHANGEME": + errors.append("bad_prometheus_password") + + if len(errors) > 0: + log.error(f"Health check errors: {errors}") + raise HTTPException(status_code=542, detail=f"health check failed") + + return { + "status": "ok", + "version": pkg_version, + "build_label": build_label, + } + + +@app.get("/") +async def root(): + return {"message": "Hello OONItarian!"} diff --git a/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py b/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py new file mode 100644 index 00000000..a25026e3 --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/routers/v1.py @@ -0,0 +1,227 @@ +""" +OONIRun link management + +https://github.com/ooni/spec/blob/master/backends/bk-005-ooni-run-v2.md +""" + +from datetime import datetime, timedelta, timezone +from typing import Optional +from urllib.parse import urlparse, urlencode, urlunsplit +import logging + +import jwt + +from fastapi import APIRouter, Depends, Query, HTTPException, Header, Path +from pydantic import Field, validator +from pydantic import EmailStr +from typing_extensions import Annotated + +from ..dependencies import get_clickhouse_client, get_ses_client + +from ..utils import ( + create_session_token, + get_account_role, + hash_email_address, + send_login_email, +) +from ..common.dependencies import get_settings, role_required +from ..common.config import Settings +from ..common.routers import BaseModel +from ..common.utils import ( + create_jwt, + decode_jwt, + get_client_token, +) + + +log = logging.getLogger(__name__) + +router = APIRouter() + +# @router.get("/api/v2/ooniauth/user-session") +# @router.post("/api/v2/ooniauth/user-session", response_model=SessionTokenCreate) +# redirect_to: ## Make this optional + + +class UserRegister(BaseModel): + email_address: EmailStr = Field( + title="email address of the user", + min_length=5, + max_length=255, + ) + redirect_to: str = Field(title="redirect to this URL") + + @validator("redirect_to") + def validate_redirect_to(cls, v): + u = urlparse(v) + if u.scheme != "https": + raise ValueError("Invalid URL") + valid_dnames = ( + "explorer.ooni.org", + "explorer.test.ooni.org", + "run.ooni.io", + "run.test.ooni.org", + "test-lists.ooni.org", + "test-lists.test.ooni.org", + ) + if u.netloc not in valid_dnames: + raise ValueError("Invalid URL", u.netloc) + + return v + + +def format_login_url(redirect_to: str, registration_token: str) -> str: + login_fqdm = urlparse(redirect_to).netloc + e = urlencode(dict(token=registration_token)) + return urlunsplit(("https", login_fqdm, "/login", e, "")) + + +class UserRegistrationResponse(BaseModel): + msg: str + + +@router.post("/v1/user_register", response_model=UserRegistrationResponse) +async def user_register( + user_register: UserRegister, + settings: Settings = Depends(get_settings), + ses_client=Depends(get_ses_client), +): + """Auth Services: start email-based user registration""" + email_address = user_register.email_address.lower() + + account_id = hash_email_address( + email_address=email_address, key=settings.account_id_hashing_key + ) + now = datetime.now(timezone.utc) + expiration = now + timedelta(days=1) + # On the backend side the registration is stateless + payload = { + "nbf": now, + "exp": expiration, + "aud": "register", + "account_id": account_id, + "email_address": email_address, + "redirect_to": user_register.redirect_to, + } + registration_token = create_jwt(payload=payload, key=settings.jwt_encryption_key) + + login_url = format_login_url( + redirect_to=user_register.redirect_to, registration_token=registration_token + ) + + log.info("sending registration token") + try: + email_id = send_login_email( + source_address=settings.email_source_address, + destination_address=email_address, + login_url=login_url, + ses_client=ses_client, + ) + log.info(f"email sent: {email_id}") + except Exception as e: + log.error(e, exc_info=True) + raise HTTPException(status_code=500, detail="Unable to send the email") + + return UserRegistrationResponse(msg="ok") + + +class SessionTokenCreate(BaseModel): + bearer: str + redirect_to: str + email_address: str + + +@router.get("/v1/user_login", response_model=SessionTokenCreate) +async def user_login( + token: Annotated[ + str, + Query(alias="k", description="JWT token with aud=register"), + ], + settings: Settings = Depends(get_settings), + db: Settings = Depends(get_clickhouse_client), +): + """Auth Services: login using a registration/login link""" + try: + dec = decode_jwt( + token=token, key=settings.jwt_encryption_key, audience="register" + ) + except ( + jwt.exceptions.MissingRequiredClaimError, + jwt.exceptions.InvalidSignatureError, + jwt.exceptions.DecodeError, + ): + raise HTTPException(401, "Invalid credentials") + except jwt.exceptions.ExpiredSignatureError: + raise HTTPException(401, "Expired token") + + log.info("user login successful") + + # Store account role in token to prevent frequent DB lookups + role = get_account_role(db=db, account_id=dec["account_id"]) or "user" + redirect_to = dec.get("redirect_to", "") + email = dec["email_address"] + + token = create_session_token( + key=settings.jwt_encryption_key, + account_id=dec["account_id"], + role=role, + session_expiry_days=settings.session_expiry_days, + login_expiry_days=settings.login_expiry_days, + ) + return SessionTokenCreate( + bearer=token, + redirect_to=redirect_to, + email_address=email, + ) + + +class SessionTokenRefresh(BaseModel): + bearer: str + + +@router.get( + "/v1/user_refresh_token", + dependencies=[Depends(role_required(["admin", "user"]))], + response_model=SessionTokenRefresh, +) +async def user_refresh_token( + settings: Settings = Depends(get_settings), + authorization: str = Header("authorization"), +): + """Auth services: refresh user token""" + tok = get_client_token( + authorization=authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + + # @role_required already checked for validity of token + assert tok is not None + + newtoken = create_session_token( + key=settings.jwt_encryption_key, + account_id=tok["account_id"], + role=tok["role"], + session_expiry_days=settings.session_expiry_days, + login_expiry_days=settings.login_expiry_days, + login_time=tok["login_time"], + ) + log.debug("user token refresh successful") + return SessionTokenRefresh(bearer=newtoken) + + +class AccountMetadata(BaseModel): + logged_in: bool + role: str + + +@router.get("/_/account_metadata") +async def get_account_metadata( + settings: Settings = Depends(get_settings), + authorization: str = Header("authorization"), +): + """Get account metadata for logged-in users""" + tok = get_client_token( + authorization=authorization, jwt_encryption_key=settings.jwt_encryption_key + ) + if not tok: + return AccountMetadata(logged_in=False, role="") + return AccountMetadata(logged_in=True, role=tok["role"]) diff --git a/ooniapi/services/ooniauth/src/ooniauth/utils.py b/ooniapi/services/ooniauth/src/ooniauth/utils.py new file mode 100644 index 00000000..e3ae4f21 --- /dev/null +++ b/ooniapi/services/ooniauth/src/ooniauth/utils.py @@ -0,0 +1,103 @@ +import hashlib +import time +from typing import Optional +from textwrap import dedent + +import sqlalchemy as sa +import boto3 + +from .common.utils import create_jwt, query_click_one_row + + +def create_session_token( + key: str, + account_id: str, + role: str, + session_expiry_days: int, + login_expiry_days: int, + login_time=None, +) -> str: + now = int(time.time()) + session_exp = now + session_expiry_days * 86400 + if login_time is None: + login_time = now + login_exp = login_time + login_expiry_days * 86400 + exp = min(session_exp, login_exp) + payload = { + "nbf": now, + "iat": now, + "exp": exp, + "aud": "user_auth", + "account_id": account_id, + "login_time": login_time, + "role": role, + } + return create_jwt(payload=payload, key=key) + + +def get_account_role(db, account_id: str) -> Optional[str]: + """Get account role from database, or None""" + query = "SELECT role FROM accounts WHERE account_id = :account_id" + query_params = dict(account_id=account_id) + r = query_click_one_row(db, sa.text(query), query_params) + return r["role"] if r else None + + +def hash_email_address(email_address: str, key: str) -> str: + return hashlib.blake2b( + email_address.encode(), key=key.encode(), digest_size=16 + ).hexdigest() + + +def send_login_email( + destination_address: str, source_address: str, login_url: str, ses_client +) -> str: + """Format and send a registration/login email""" + body_text = dedent( + f""" + Welcome to OONI. + Please login by following {login_url} + The link can be used on multiple devices and will expire in 24 hours. + """ + ) + + body_html = dedent( + f""" + + + +

Welcome to OONI

+

+ Please login here +

+

The link can be used on multiple devices and will expire in 24 hours.

+ + + """ + ) + + response = ses_client.send_email( + Destination={ + "ToAddresses": [ + destination_address, + ], + }, + Message={ + "Body": { + "Html": { + "Charset": "UTF-8", + "Data": body_html, + }, + "Text": { + "Charset": "UTF-8", + "Data": body_text, + }, + }, + "Subject": { + "Charset": "UTF-8", + "Data": "OONI Account activation email", + }, + }, + Source=source_address, + ) + return response["MessageId"] diff --git a/ooniapi/services/ooniauth/tests/__init__.py b/ooniapi/services/ooniauth/tests/__init__.py new file mode 100644 index 00000000..632b5b71 --- /dev/null +++ b/ooniapi/services/ooniauth/tests/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2024-present Arturo Filastò +# +# SPDX-License-Identifier: MIT diff --git a/ooniapi/services/ooniauth/tests/conftest.py b/ooniapi/services/ooniauth/tests/conftest.py new file mode 100644 index 00000000..55c26588 --- /dev/null +++ b/ooniapi/services/ooniauth/tests/conftest.py @@ -0,0 +1,120 @@ +from unittest.mock import MagicMock +import pytest + +import time +import jwt + +from fastapi.testclient import TestClient + +from ooniauth.common.config import Settings +from ooniauth.common.dependencies import get_settings +from ooniauth.dependencies import get_ses_client, get_clickhouse_client +from ooniauth.utils import hash_email_address +from ooniauth.main import app + + +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings + + +@pytest.fixture +def client_with_bad_settings(): + app.dependency_overrides[get_settings] = make_override_get_settings( + postgresql_url="postgresql://bad:bad@localhost/bad" + ) + + client = TestClient(app) + yield client + + +@pytest.fixture +def user_email(): + return "dev+useraccount@ooni.org" + + +@pytest.fixture +def admin_email(): + return "dev+adminaccount@ooni.org" + + +@pytest.fixture +def jwt_encryption_key(): + return "super_secure" + + +@pytest.fixture +def prometheus_password(): + return "super_secure" + + +@pytest.fixture +def account_id_hashing_key(): + return "super_secure" + + +@pytest.fixture +def email_source_address(): + return "admin+sourceemail@ooni.org" + + +@pytest.fixture +def valid_redirect_to_url(): + return "https://explorer.ooni.org" + + +@pytest.fixture +def mock_ses_client(): + mock = MagicMock() + app.dependency_overrides[get_ses_client] = lambda: mock + yield mock + + +@pytest.fixture +def mock_misconfigured_ses_client(): + mock = MagicMock() + mock.send_email.side_effect = Exception("failing to send an email") + app.dependency_overrides[get_ses_client] = lambda: mock + yield mock + + +@pytest.fixture +def client( + mock_ses_client, + admin_email, + jwt_encryption_key, + account_id_hashing_key, + prometheus_password, + email_source_address, +): + app.dependency_overrides[get_settings] = make_override_get_settings( + jwt_encryption_key=jwt_encryption_key, + prometheus_metrics_password=prometheus_password, + email_source_address=email_source_address, + account_id_hashing_key=account_id_hashing_key, + aws_access_key_id="ITSCHANGED", + aws_secret_access_key="ITSCHANGED", + ) + mock_clickhouse = MagicMock() + mock_clickhouse.execute = MagicMock() + + # rows, coldata = q + # coldata = [("name", "type")] + def mock_execute(query, query_params, with_column_types, settings): + assert with_column_types == True + print(settings) + assert query.startswith("SELECT role FROM") + if query_params["account_id"] == hash_email_address( + email_address=admin_email, key=account_id_hashing_key + ): + return [("admin",)], [("role", "String")] + + return [("user",)], [("role", "String")] + + mock_clickhouse.execute = mock_execute + app.dependency_overrides[get_clickhouse_client] = lambda: mock_clickhouse + + client = TestClient(app) + yield client diff --git a/ooniapi/services/ooniauth/tests/run_live_test.py b/ooniapi/services/ooniauth/tests/run_live_test.py new file mode 100644 index 00000000..cfada8d5 --- /dev/null +++ b/ooniapi/services/ooniauth/tests/run_live_test.py @@ -0,0 +1,14 @@ +import httpx + + +def main(): + d = dict(email_address="arturo@ooni.org", redirect_to="https://explorer.ooni.org/") + r = httpx.post("https://api.dev.ooni.io/api/v1/user_register", json=d) + print(r.text) + j = r.json() + print(j) + assert r.status_code == 200 + + +if __name__ == "__main__": + main() diff --git a/ooniapi/services/ooniauth/tests/test_auth.py b/ooniapi/services/ooniauth/tests/test_auth.py new file mode 100644 index 00000000..e8768fce --- /dev/null +++ b/ooniapi/services/ooniauth/tests/test_auth.py @@ -0,0 +1,196 @@ +""" +Integration test for Auth API + +Warning: this test runs against a real database and SMTP + +Lint using: + black -t py37 -l 100 --fast ooniapi/tests/integ/test_probe_services.py + +Test using: + pytest-3 -s --show-capture=no ooniapi/tests/integ/test_integration_auth.py +""" + +from urllib.parse import parse_qs, urlparse +from ooniauth.common.utils import decode_jwt +from ooniauth.main import app +from freezegun import freeze_time + + +def register(client, email_address, mock_ses_client, valid_redirect_to_url): + d = dict(email_address=email_address, redirect_to=valid_redirect_to_url) + r = client.post("/api/v1/user_register", json=d) + j = r.json() + assert r.status_code == 200 + assert j == {"msg": "ok"} + + from html.parser import HTMLParser + + class AHrefParser(HTMLParser): + links = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + for attr in attrs: + if attr[0] == "href": + self.links.append(attr[1]) + + mock_send_email = mock_ses_client.send_email + assert ( + mock_send_email.call_args.kwargs["Destination"]["ToAddresses"][0] + == email_address + ) + html_message = mock_send_email.call_args.kwargs["Message"]["Body"]["Html"]["Data"] + assert "source" in mock_send_email.call_args.kwargs["Source"] + parser = AHrefParser() + parser.feed(html_message) + parser.close() + + login_link = parser.links[0] + token = parse_qs(urlparse(login_link).query)["token"][0] + assert len(token) > 300 + return token + + +def register_and_login(client, email_address, mock_ses_client, valid_redirect_to_url): + token = register( + client, + email_address=email_address, + mock_ses_client=mock_ses_client, + valid_redirect_to_url=valid_redirect_to_url, + ) + r = client.get(f"/api/v1/user_login?k={token}") + j = r.json() + assert r.status_code == 200, j + assert j["redirect_to"] == valid_redirect_to_url + assert "bearer" in j + return {"Authorization": "Bearer " + j["bearer"]} + + +def test_login_user_bogus_token(client): + r = client.get("/api/v1/user_login?k=BOGUS") + assert r.status_code == 401 + # Note the key changed from "error" to "detail" + assert r.json() == {"detail": "Invalid credentials"} + + +def test_user_register_non_valid_email(client, valid_redirect_to_url): + d = dict( + email_address="nick@localhost", redirect_to=valid_redirect_to_url + ) # no FQDN + r = client.post("/api/v1/user_register", json=d) + assert r.status_code == 422 + j = r.json() + assert j["detail"][0]["loc"] == ["body", "email_address"] + + +def test_user_register_non_valid_redirect(client, valid_redirect_to_url): + d = dict( + email_address="nick@a.org", redirect_to="https://BOGUS.example.com" + ) # bogus fqdn + r = client.post("/api/v1/user_register", json=d) + assert r.status_code == 422 + + d = dict( + email_address="nick@a.org", + redirect_to=valid_redirect_to_url.replace("https", "http"), + ) # bogus fqdn + r = client.post("/api/v1/user_register", json=d) + assert r.status_code == 422 + + +def test_user_register_missing_redirect(client, valid_redirect_to_url): + d = dict(email_address="nick@a.org", redirect_to=valid_redirect_to_url) + r = client.post("/api/v1/user_register", json=d) + assert r.status_code == 200 + + +def test_user_refresh(client, mock_ses_client, user_email, valid_redirect_to_url): + r = client.get( + "/api/v1/user_refresh_token", headers={"Authorization": "Bearer invalidtoken"} + ) + assert r.status_code != 200 + j = r.json() + print(j) + + h = register_and_login(client, user_email, mock_ses_client, valid_redirect_to_url) + j = client.get("/api/_/account_metadata", headers=h).json() + assert j["logged_in"] == True + assert j["role"] == "user" + + j = client.get("/api/v1/user_refresh_token", headers=h).json() + assert len(j["bearer"]) > 100 + + +def test_user_register_and_refresh( + client, mock_ses_client, user_email, valid_redirect_to_url +): + j = client.get("/api/_/account_metadata").json() + assert j["logged_in"] == False + h = register_and_login(client, user_email, mock_ses_client, valid_redirect_to_url) + j = client.get("/api/_/account_metadata", headers=h).json() + assert j["logged_in"] == True + assert j["role"] == "user" + + +def test_user_register_misconfigured_email( + client, mock_misconfigured_ses_client, user_email, valid_redirect_to_url +): + d = dict(email_address=user_email, redirect_to=valid_redirect_to_url) + r = client.post("/api/v1/user_register", json=d) + assert r.status_code != 200 + assert isinstance(mock_misconfigured_ses_client.send_email.side_effect, Exception) + + +def test_user_register_and_get_metadata( + client, mock_ses_client, user_email, valid_redirect_to_url +): + r = client.get("/api/_/account_metadata") + j = r.json() + assert j["logged_in"] == False + h = register_and_login(client, user_email, mock_ses_client, valid_redirect_to_url) + r = client.get("/api/_/account_metadata", headers=h) + j = r.json() + assert j["role"] == "user" + assert j["logged_in"] == True + + +def test_admin_register_and_get_metadata( + client, mock_ses_client, admin_email, valid_redirect_to_url +): + r = client.get("/api/_/account_metadata") + j = r.json() + assert j["logged_in"] == False + h = register_and_login(client, admin_email, mock_ses_client, valid_redirect_to_url) + r = client.get("/api/_/account_metadata", headers=h) + j = r.json() + assert j["role"] == "admin" + assert j["logged_in"] == True + + +def test_user_register_timetravel( + client, mock_ses_client, user_email, valid_redirect_to_url +): + with freeze_time("1990-01-01"): + token = register( + client, + email_address=user_email, + mock_ses_client=mock_ses_client, + valid_redirect_to_url=valid_redirect_to_url, + ) + r = client.get(f"/api/v1/user_login?k={token}") + j = r.json() + assert r.status_code == 200, j + assert len(j["bearer"]) > 100 + assert j["email_address"] == user_email + + h = register_and_login( + client, user_email, mock_ses_client, valid_redirect_to_url + ) + j = client.get("/api/_/account_metadata", headers=h).json() + assert j["logged_in"] == True + assert j["role"] == "user" + + with freeze_time("1995-01-01"): + r = client.get(f"/api/v1/user_login?k={token}") + j = r.json() + assert r.status_code != 200, j diff --git a/ooniapi/services/ooniauth/tests/test_main.py b/ooniapi/services/ooniauth/tests/test_main.py new file mode 100644 index 00000000..58e14244 --- /dev/null +++ b/ooniapi/services/ooniauth/tests/test_main.py @@ -0,0 +1,55 @@ +import pytest + +import httpx +from fastapi.testclient import TestClient +from ooniauth.main import lifespan, app, pkg_version +from ooniauth.common.config import Settings +from ooniauth.common.dependencies import get_settings + + +def test_index(client): + r = client.get("/") + j = r.json() + assert r.status_code == 200 + + +def test_version(client): + r = client.get("/version") + j = r.json() + assert j["version"] == pkg_version + assert len(j["package_name"]) > 1 + + +def test_health_good(client): + r = client.get("/health") + j = r.json() + assert j["status"] == "ok", j + + +def test_health_bad(client_with_bad_settings): + r = client_with_bad_settings.get("/health") + j = r.json() + assert r.status_code != 200 + + +def make_override_get_settings(**kw): + def override_get_settings(): + return Settings(**kw) + + return override_get_settings + + +@pytest.mark.asyncio +async def test_lifecycle(prometheus_password): + app.dependency_overrides[get_settings] = make_override_get_settings( + prometheus_metrics_password=prometheus_password, + ) + + async with lifespan(app) as ls: + client = TestClient(app) + r = client.get("/metrics") + assert r.status_code == 401 + + auth = httpx.BasicAuth(username="prom", password=prometheus_password) + r = client.get("/metrics", auth=auth) + assert r.status_code == 200, r.text diff --git a/ooniapi/services/oonirun/src/oonirun/main.py b/ooniapi/services/oonirun/src/oonirun/main.py index ef4ef96e..aeccf355 100644 --- a/ooniapi/services/oonirun/src/oonirun/main.py +++ b/ooniapi/services/oonirun/src/oonirun/main.py @@ -11,7 +11,9 @@ from . import models from .routers import oonirun -from .dependencies import get_postgresql_session, get_settings +from .dependencies import get_postgresql_session +from .common.dependencies import get_settings +from .common.version import get_build_label, get_pkg_version from .common.version import get_build_label, get_pkg_version from .common.metrics import mount_metrics diff --git a/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py b/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py index d216dc88..34044274 100644 --- a/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py +++ b/ooniapi/services/oonirun/src/oonirun/routers/oonirun.py @@ -12,11 +12,11 @@ from sqlalchemy.orm import Session from fastapi import APIRouter, Depends, Query, HTTPException, Header, Path from pydantic import computed_field, Field, validator -from pydantic import BaseModel as PydandicBaseModel from typing_extensions import Annotated from .. import models +from ..common.routers import BaseModel from ..common.dependencies import get_settings, role_required from ..common.utils import ( get_client_role, @@ -25,17 +25,6 @@ ) from ..dependencies import get_postgresql_session -ISO_FORMAT_DATETIME = "%Y-%m-%dT%H:%M:%S.%fZ" -ISO_FORMAT_DATE = "%Y-%m-%d" - - -class BaseModel(PydandicBaseModel): - class Config: - json_encoders = { - datetime: lambda v: v.strftime(ISO_FORMAT_DATETIME), - date: lambda v: v.strftime(ISO_FORMAT_DATE), - } - log = logging.getLogger(__name__)