From 13bff04ba4754e41a80cc3c9f621e277d53b6682 Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Mon, 15 Jul 2024 16:03:59 -0700 Subject: [PATCH] bug-1908018: drop support for AWS --- .github/workflows/build-and-push.yml | 15 -- Makefile | 2 +- antenna/app.py | 2 - antenna/crashmover.py | 2 +- antenna/ext/fs/crashstorage.py | 2 +- antenna/ext/s3/__init__.py | 3 - antenna/ext/s3/connection.py | 248 ------------------- antenna/ext/s3/crashstorage.py | 165 ------------- antenna/ext/sqs/__init__.py | 3 - antenna/ext/sqs/crashpublish.py | 163 ------------- antenna/health_resource.py | 10 - bin/run_setup.sh | 15 +- bin/run_tests.sh | 1 - bin/s3_cli.py | 131 ---------- bin/sqs_cli.py | 169 ------------- docker-compose.yml | 17 -- docker/config/local_dev.env | 20 +- docker/config/test.env | 5 +- docs/configuration.rst | 59 +---- docs/deploy.rst | 30 +-- docs/dev.rst | 41 ++-- docs/overview.rst | 21 +- pyproject.toml | 7 +- requirements.in | 2 - requirements.txt | 26 +- systemtest/README.rst | 8 +- systemtest/conftest.py | 181 +------------- systemtest/test_content_length.py | 10 +- systemtest/test_dockerflow.py | 5 +- systemtest/test_env.sh | 9 +- systemtest/test_post_crash.py | 4 +- testlib/mini_poster.py | 2 +- testlib/s3mock.py | 352 --------------------------- tests/conftest.py | 64 ----- tests/test_bin.py | 20 -- tests/test_health_resource.py | 46 +--- tests/test_s3_crashstorage.py | 321 ------------------------ tests/test_sentry.py | 1 - tests/test_sqs_crashpublish.py | 134 ---------- 39 files changed, 79 insertions(+), 2237 deletions(-) delete mode 100644 antenna/ext/s3/__init__.py delete mode 100644 antenna/ext/s3/connection.py delete mode 100644 antenna/ext/s3/crashstorage.py delete mode 100644 antenna/ext/sqs/__init__.py delete mode 100644 antenna/ext/sqs/crashpublish.py delete mode 100755 bin/s3_cli.py delete mode 100755 bin/sqs_cli.py delete mode 100644 testlib/s3mock.py delete mode 100644 tests/test_s3_crashstorage.py delete mode 100644 tests/test_sqs_crashpublish.py diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 3f15dcf4..6042920d 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -47,27 +47,12 @@ jobs: - name: Run tests run: | make my.env - docker compose up --detach --no-color \ - localstack \ - statsd \ - fakesentry docker compose run --rm ci shell ./bin/run_tests.sh - name: Run systemtest run: | docker compose run --rm ci-web shell ./bin/run_setup.sh docker compose up --detach --wait --wait-timeout=10 ci-web docker compose run --rm ci-web shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv' - - name: Run systemtest with pubsub and gcs - run: | - echo 'CRASHMOVER_CRASHPUBLISH_CLASS=antenna.ext.pubsub.crashpublish.PubSubCrashPublish' >> my.env - echo 'CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage' >> my.env - docker compose run --rm ci-web shell ./bin/run_setup.sh - docker compose up --detach --wait --wait-timeout=10 ci-web - # Use -m "not aws" to select gcp and unmarked tests - docker compose run --rm ci-web shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv -m "not aws"' - # remove config on last two lines - sed '$d' -i my.env - sed '$d' -i my.env - name: Set Docker image tag to "latest" for updates of the main branch if: github.ref == 'refs/heads/main' diff --git a/Makefile b/Makefile index 5e09417e..a8ddec35 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ my.env: .PHONY: build build: my.env ## | Build docker images. ${DC} --progress plain build ${DOCKER_BUILD_OPTS} --build-arg userid=${ANTENNA_UID} --build-arg groupid=${ANTENNA_GID} deploy-base - ${DC} --progress plain build fakesentry gcs-emulator localstack statsd + ${DC} --progress plain build fakesentry gcs-emulator statsd touch .docker-build .PHONY: setup diff --git a/antenna/app.py b/antenna/app.py index d2ceb489..739b5561 100644 --- a/antenna/app.py +++ b/antenna/app.py @@ -20,7 +20,6 @@ from fillmore.scrubber import Scrubber, Rule, SCRUB_RULES_DEFAULT import sentry_sdk from sentry_sdk.integrations.atexit import AtexitIntegration -from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.integrations.dedupe import DedupeIntegration from sentry_sdk.integrations.excepthook import ExcepthookIntegration from sentry_sdk.integrations.modules import ModulesIntegration @@ -78,7 +77,6 @@ def configure_sentry(app_config): default_integrations=False, integrations=[ AtexitIntegration(), - Boto3Integration(), ExcepthookIntegration(), DedupeIntegration(), StdlibIntegration(), diff --git a/antenna/crashmover.py b/antenna/crashmover.py index 2e57409c..afa481ae 100644 --- a/antenna/crashmover.py +++ b/antenna/crashmover.py @@ -49,7 +49,7 @@ class CrashMover: For example:: - CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.s3.crashstorage.S3CrashStorage + CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage """ diff --git a/antenna/ext/fs/crashstorage.py b/antenna/ext/fs/crashstorage.py index 307cfff2..c9aac9be 100644 --- a/antenna/ext/fs/crashstorage.py +++ b/antenna/ext/fs/crashstorage.py @@ -21,7 +21,7 @@ class FSCrashStorage(CrashStorageBase): """Save raw crash files to the file system. This generates a tree something like this which mirrors what we do - on S3: + on GCS: :: diff --git a/antenna/ext/s3/__init__.py b/antenna/ext/s3/__init__.py deleted file mode 100644 index 448bb865..00000000 --- a/antenna/ext/s3/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/antenna/ext/s3/connection.py b/antenna/ext/s3/connection.py deleted file mode 100644 index 2883b888..00000000 --- a/antenna/ext/s3/connection.py +++ /dev/null @@ -1,248 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - - -import io -import logging -import random -import uuid - -import boto3 -from botocore.client import ClientError, Config -from everett.manager import Option - -from antenna.util import retry - - -LOGGER = logging.getLogger(__name__) - - -class KeyNotFound(Exception): - pass - - -def generate_test_filepath(): - """Generate a unique-ish test filepath.""" - return "test/testfile-%s.txt" % uuid.uuid4() - - -def wait_times_connect(): - """Return generator for wait times between failed connection attempts. - - We have this problem where we're binding IAM credentials to the EC2 node - and on startup when boto3 goes to get the credentials, it fails for some - reason and then degrades to hitting the https://s3..amazonaws.net/ - endpoint and then fails because that's not a valid endpoint. - - This sequence increases the wait times and adds some jitter. - - """ - for i in [5] * 5: - yield i + random.uniform(-2, 2) # noqa: S311 - - -def wait_times_save(): - """Return generator for wait times between failed save attempts. - - This waits 2 seconds between failed save attempts for 5 iterations and then - gives up. - - """ - yield from [2, 2, 2, 2, 2] - - -class S3Connection: - """Connection object for S3. - - **Credentials and permissions** - - When configuring credentials for this connection object, you can do one of two - things: - - 1. provide ``ACCESS_KEY`` and ``SECRET_ACCESS_KEY`` in the configuration, OR - 2. use one of the other methods described in the boto3 docs - https://boto3.readthedocs.io/en/latest/guide/configuration.html#configuring-credentials - - - The AWS credentials that Antenna is configured with must have the following - Amazon S3 permissions: - - * ``s3:ListBucket`` - - Antenna periodically checks its health and during that health check, it - will HEAD the S3 Bucket. This requires ``s3:ListBucket`` permissions. - - * ``s3:PutObject`` - - This permission is used to save items to the bucket. - - Additionally, at startup, Antenna will attempt to save a test file to the - bucket. If that fails, then this will raise an error and will halt - startup. - - - **Retrying saves** - - When saving crashes, this connection will retry saving several times. Then give up. - - """ - - KeyNotFound = KeyNotFound - - class Config: - access_key = Option( - default="", - alternate_keys=["root:aws_access_key_id"], - doc=( - "AWS access key. You can also specify AWS_ACCESS_KEY_ID which is " - "the env var used by boto3." - ), - ) - secret_access_key = Option( - default="", - alternate_keys=["root:aws_secret_access_key"], - doc=( - "AWS secret access key. You can also specify AWS_SECRET_ACCESS_KEY " - "which is the env var used by boto3." - ), - ) - region = Option( - default="us-west-2", - alternate_keys=["root:s3_region"], - doc="AWS region to connect to. For example, ``us-west-2``", - ) - endpoint_url = Option( - default="", - alternate_keys=["root:s3_endpoint_url"], - doc=( - "endpoint_url to connect to; None if you are connecting to AWS. For " - "example, ``http://localhost:4569/``." - ), - ) - bucket_name = Option( - doc=( - "AWS S3 bucket to save to. Note that the bucket must already have been " - "created and must be in the region specified by ``region``." - ), - ) - - def __init__(self, config): - self.config = config.with_options(self) - self.bucket = self.config("bucket_name") - self.client = self._build_client() - - @retry( - retryable_exceptions=[ - # FIXME(willkg): Seems like botocore always raises ClientError - # which is unhelpful for granularity purposes. - ClientError, - # This raises a ValueError "invalid endpoint" if it has problems - # getting the s3 credentials and then tries "s3..amazonaws.com"--we - # want to retry that, too. - ValueError, - ], - wait_time_generator=wait_times_connect, - module_logger=LOGGER, - ) - def _build_client(self): - # Either they provided ACCESS_KEY and SECRET_ACCESS_KEY in which case - # we use those, or they didn't in which case boto3 pulls credentials - # from one of a myriad of other places. - # https://boto3.readthedocs.io/en/latest/guide/configuration.html#configuring-credentials - session_kwargs = {} - if self.config("access_key") and self.config("secret_access_key"): - session_kwargs["aws_access_key_id"] = self.config("access_key") - session_kwargs["aws_secret_access_key"] = self.config("secret_access_key") - session = boto3.session.Session(**session_kwargs) - - kwargs = { - "service_name": "s3", - "region_name": self.config("region"), - # NOTE(willkg): We use path-style because that lets us have dots in - # our bucket names and use SSL. - "config": Config(s3={"addressing_style": "path"}), - } - if self.config("endpoint_url"): - kwargs["endpoint_url"] = self.config("endpoint_url") - - return session.client(**kwargs) - - def verify_write_to_bucket(self): - """Verify S3 bucket exists and can be written to. - - This will do multiple attempts and then give up and throw an exception. - - """ - self.client.upload_fileobj( - Fileobj=io.BytesIO(b"test"), - Bucket=self.bucket, - Key=generate_test_filepath(), - ) - - def check_health(self, state): - """Check S3 connection health.""" - try: - # HEAD the bucket to verify S3 is up and we can connect to it. - self.client.head_bucket(Bucket=self.bucket) - except Exception as exc: - state.add_error("S3Connection", repr(exc)) - - @retry( - retryable_exceptions=[ - # FIXME(willkg): Seems like botocore always raises ClientError - # which is unhelpful for granularity purposes. - ClientError - ], - wait_time_generator=wait_times_save, - module_logger=LOGGER, - ) - def save_file(self, path, data): - """Save a single file to S3. - - This will retry a handful of times in short succession so as to deal - with some amount of fishiness. After that, the caller should retry - saving after a longer period of time. - - :arg str path: the path to save to - - :arg bytes data: the data to save - - :raises botocore.exceptions.ClientError: connection issues, permissions - issues, bucket is missing, etc. - - """ - if not isinstance(data, bytes): - raise TypeError("data argument must be bytes") - - self.client.upload_fileobj( - Fileobj=io.BytesIO(data), Bucket=self.bucket, Key=path - ) - - @retry( - retryable_exceptions=[ - # FIXME(willkg): Seems like botocore always raises ClientError - # which is unhelpful for granularity purposes. - ClientError - ], - wait_time_generator=wait_times_save, - module_logger=LOGGER, - ) - def load_file(self, path): - """Load a file from S3. - - This will retry a handful of times in short succession so as to deal - with some amount of fishiness. After that, the caller should retry - loading after a longer period of time. - - :arg str path: the path to save to - - :raises botocore.exceptions.ClientError: connection issues, permissions - issues, bucket is missing, etc. - - """ - try: - resp = self.client.get_object(Bucket=self.bucket, Key=path) - return resp["Body"].read() - except self.client.exceptions.NoSuchKey as exc: - raise KeyNotFound(f"{path} not found") from exc diff --git a/antenna/ext/s3/crashstorage.py b/antenna/ext/s3/crashstorage.py deleted file mode 100644 index e40c973b..00000000 --- a/antenna/ext/s3/crashstorage.py +++ /dev/null @@ -1,165 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import json - -from everett.manager import Option, parse_class - -from antenna.app import register_for_verification -from antenna.crashmover import CrashReport -from antenna.ext.crashstorage_base import CrashStorageBase, CrashIDNotFound -from antenna.util import get_date_from_crash_id, json_ordered_dumps - - -class S3CrashStorage(CrashStorageBase): - """Save raw crash files to S3. - - This will save raw crash files to S3 in a pseudo-tree something like this: - - :: - - - v1/ - dump_names/ - - / - - raw_crash/ - / - - - """ - - class Config: - connection_class = Option( - default="antenna.ext.s3.connection.S3Connection", - parser=parse_class, - doc="S3 connection class to use", - ) - - def __init__(self, config): - self.config = config.with_options(self) - self.connection = self.config("connection_class")(config) - register_for_verification(self.verify_write_to_bucket) - - def get_components(self): - """Return map of namespace -> component for traversing component tree.""" - return {"": self.connection} - - def verify_write_to_bucket(self): - """Verify S3 bucket exists and can be written to.""" - self.connection.verify_write_to_bucket() - - def get_runtime_config(self, namespace=None): - """Return generator for items in runtime configuration.""" - yield from super().get_runtime_config(namespace) - - yield from self.connection.get_runtime_config(namespace) - - def check_health(self, state): - """Check connection health.""" - self.connection.check_health(state) - - def _get_raw_crash_path(self, crash_id): - return "v1/raw_crash/{date}/{crash_id}".format( - date=get_date_from_crash_id(crash_id), - crash_id=crash_id, - ) - - def _get_dump_names_path(self, crash_id): - return f"v1/dump_names/{crash_id}" - - def _get_dump_name_path(self, crash_id, dump_name): - # NOTE(willkg): This is something that Socorro collector did. I'm not - # really sure why, but in order to maintain backwards compatability, we - # need to keep doing it. - if dump_name in (None, "", "upload_file_minidump"): - dump_name = "dump" - - return "v1/{dump_name}/{crash_id}".format( - dump_name=dump_name, crash_id=crash_id - ) - - def save_raw_crash(self, crash_id, raw_crash): - """Save the raw crash and related dumps. - - .. Note:: - - If you're saving the raw crash and dumps, make sure to save the raw - crash last. - - :arg str crash_id: The crash id as a string. - :arg dict raw_crash: dict The raw crash as a dict. - - :raises botocore.exceptions.ClientError: connection issues, permissions - issues, bucket is missing, etc. - - """ - # FIXME(willkg): self.connection.save_file raises a - # botocore.exceptions.ClientError if the perms aren't right. That needs - # to surface to "this node is not healthy". - - # Save raw_crash - self.connection.save_file( - self._get_raw_crash_path(crash_id), - json_ordered_dumps(raw_crash).encode("utf-8"), - ) - - def save_dumps(self, crash_id, dumps): - """Save dump data. - - :arg str crash_id: The crash id - :arg dict dumps: dump name -> dump - - :raises botocore.exceptions.ClientError: connection issues, permissions - issues, bucket is missing, etc. - - """ - # Save dump_names even if there are no dumps - self.connection.save_file( - self._get_dump_names_path(crash_id), - json_ordered_dumps(list(sorted(dumps.keys()))).encode("utf-8"), - ) - - # Save dumps - for dump_name, dump in dumps.items(): - self.connection.save_file( - self._get_dump_name_path(crash_id, dump_name), dump - ) - - def save_crash(self, crash_report): - """Save crash data.""" - crash_id = crash_report.crash_id - raw_crash = crash_report.raw_crash - dumps = crash_report.dumps - - # Save dumps first - self.save_dumps(crash_id, dumps) - - # Save raw crash - self.save_raw_crash(crash_id, raw_crash) - - def load_crash(self, crash_id): - """Load crash data.""" - raw_crash = {} - dumps = {} - - raw_crash_key = self._get_raw_crash_path(crash_id) - try: - raw_crash = json.loads(self.connection.load_file(raw_crash_key)) - except self.connection.KeyNotFound as exc: - raise CrashIDNotFound(f"{crash_id} not found") from exc - - dump_names_path = self._get_dump_names_path(crash_id) - dump_names = json.loads(self.connection.load_file(dump_names_path)) - - for dump_name in dump_names: - dump_name_path = self._get_dump_name_path(crash_id, dump_name) - dumps[dump_name] = self.connection.load_file(dump_name_path) - - return CrashReport( - crash_id=crash_id, - raw_crash=raw_crash, - dumps=dumps, - ) diff --git a/antenna/ext/sqs/__init__.py b/antenna/ext/sqs/__init__.py deleted file mode 100644 index 448bb865..00000000 --- a/antenna/ext/sqs/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/antenna/ext/sqs/crashpublish.py b/antenna/ext/sqs/crashpublish.py deleted file mode 100644 index 18c83d47..00000000 --- a/antenna/ext/sqs/crashpublish.py +++ /dev/null @@ -1,163 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import logging -import random - -import boto3 -from botocore.client import ClientError -from everett.manager import Option - -from antenna.app import register_for_verification -from antenna.ext.crashpublish_base import CrashPublishBase -from antenna.util import retry - - -LOGGER = logging.getLogger(__name__) - - -def wait_times_connect(): - """Return generator for wait times with jitter between failed connection attempts.""" - for i in [5] * 5: - yield i + random.uniform(-2, 2) # noqa: S311 - - -class SQSCrashPublish(CrashPublishBase): - """Publisher to AWS SQS. - - **Required AWS SQS things** - - When configuring credentials for this crashpublish object, you can do one of two - things: - - 1. provide ``ACCESS_KEY`` and ``SECRET_ACCESS_KEY`` in the configuration, OR - 2. use one of the other methods described in the boto3 docs - https://boto3.readthedocs.io/en/latest/guide/configuration.html#configuring-credentials - - You also need to create an AWS SQS standard queue with the following settings: - - ========================== ========= - Setting Value - ========================== ========= - Default Visibility Timeout 5 minutes - Message Retention Period *default* - Maximum Message Size *default* - Delivery Delay *default* - Receive Message Wait Time *default* - ========================== ========= - - The AWS credentials that Antenna is configured with must have the following - Amazon SQS permissions on the SQS queue you created: - - * ``sqs:GetQueueUrl`` - - Antenna needs to convert a queue name to a queue url. This requires the - ``sqs:GetQueueUrl`` - - * ``sqs:SendMessage`` - - Antenna sends messages to a queue--this is how it publishes crash ids. - This requires the ``sqs:SendMessage`` permission. - - If something isn't configured correctly, then Antenna may not start. - - - **Verification** - - This component verifies that it can publish to the queue by publishing a - fake crash id of ``test``. Downstream consumers should ignore these. - - """ - - class Config: - access_key = Option( - default="", - alternate_keys=["root:aws_access_key_id"], - doc=( - "AWS SQS access key. You can also specify AWS_ACCESS_KEY_ID which is " - "the env var used by boto3." - ), - ) - secret_access_key = Option( - default="", - alternate_keys=["root:aws_secret_access_key"], - doc=( - "AWS SQS secret access key. You can also specify AWS_SECRET_ACCESS_KEY " - "which is the env var used by boto3." - ), - ) - region = Option( - default="us-west-2", - alternate_keys=["root:s3_region"], - doc="AWS region to connect to. For example, ``us-west-2``", - ) - endpoint_url = Option( - default="", - alternate_keys=["root:s3_endpoint_url"], - doc=( - "endpoint_url to connect to; None if you are connecting to AWS. For " - "example, ``http://localhost:4569/``." - ), - ) - queue_name = Option(doc="The AWS SQS queue name.") - - def __init__(self, config): - super().__init__(config) - - self.queue_name = self.config("queue_name") - self.client = self._build_client() - self.queue_url = self.client.get_queue_url(QueueName=self.queue_name)[ - "QueueUrl" - ] - - register_for_verification(self.verify_queue) - - @retry( - retryable_exceptions=[ - # FIXME(willkg): Seems like botocore always raises ClientError - # which is unhelpful for granularity purposes. - ClientError, - # This raises a ValueError "invalid endpoint" if it has problems - # getting the s3 credentials and then tries "s3..amazonaws.com"--we - # want to retry that, too. - ValueError, - ], - wait_time_generator=wait_times_connect, - module_logger=LOGGER, - ) - def _build_client(self): - # Either they provided ACCESS_KEY and SECRET_ACCESS_KEY in which case - # we use those, or they didn't in which case boto3 pulls credentials - # from one of a myriad of other places. - # https://boto3.readthedocs.io/en/latest/guide/configuration.html#configuring-credentials - session_kwargs = {} - if self.config("access_key") and self.config("secret_access_key"): - session_kwargs["aws_access_key_id"] = self.config("access_key") - session_kwargs["aws_secret_access_key"] = self.config("secret_access_key") - session = boto3.session.Session(**session_kwargs) - - kwargs = { - "service_name": "sqs", - "region_name": self.config("region"), - } - if self.config("endpoint_url"): - kwargs["endpoint_url"] = self.config("endpoint_url") - - return session.client(**kwargs) - - def verify_queue(self): - """Verify queue can be published to by publishing fake crash id.""" - self.client.send_message(QueueUrl=self.queue_url, MessageBody="test") - - def check_health(self, state): - """Check AWS SQS connection health.""" - try: - self.client.get_queue_url(QueueName=self.queue_name) - except Exception as exc: - state.add_error("SQSCrashPublish", repr(exc)) - - def publish_crash(self, crash_report): - """Publish a crash id to an AWS SQS queue.""" - crash_id = crash_report.crash_id - self.client.send_message(QueueUrl=self.queue_url, MessageBody=crash_id) diff --git a/antenna/health_resource.py b/antenna/health_resource.py index 02938858..c2b7d4d4 100644 --- a/antenna/health_resource.py +++ b/antenna/health_resource.py @@ -4,7 +4,6 @@ from collections import OrderedDict import json -import os import falcon @@ -32,15 +31,6 @@ def on_get(self, req, resp): """Implement GET HTTP request.""" METRICS.incr("collector.health.version.count") version_info = get_version_info(self.basedir) - # FIXME(willkg): there's no cloud provider environment variable to use, so - # we'll cheat and look at whether there's a "gcs" in - # CRASHMOVER_CRASHSTORAGE_CLASS; this is termporary and we can remove it - # once we've finished the GCP migration - version_info["cloud"] = ( - "GCP" - if "gcs" in os.environ.get("CRASHMOVER_CRASHSTORAGE_CLASS", "") - else "AWS" - ) resp.content_type = "application/json; charset=utf-8" resp.status = falcon.HTTP_200 diff --git a/bin/run_setup.sh b/bin/run_setup.sh index 4f00084e..9432aefe 100755 --- a/bin/run_setup.sh +++ b/bin/run_setup.sh @@ -6,30 +6,17 @@ # Usage: bin/run_setup.sh # -# Wipes and then sets up Pub/Sub and S3 services. +# Wipes and then sets up Pub/Sub and GCS services. # # This should be called from inside a container. set -euo pipefail -# Wait for services to be ready (both have the same endpoint url) -echo "Waiting for ${CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL} ..." -urlwait "${CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL}" 15 - -echo "Delete and create S3 bucket..." -python ./bin/s3_cli.py delete "${CRASHMOVER_CRASHSTORAGE_BUCKET_NAME}" -python ./bin/s3_cli.py create "${CRASHMOVER_CRASHSTORAGE_BUCKET_NAME}" -python ./bin/s3_cli.py list_buckets - echo "Delete and create GCS bucket..." python ./bin/gcs_cli.py delete "${CRASHMOVER_CRASHSTORAGE_BUCKET_NAME}" python ./bin/gcs_cli.py create "${CRASHMOVER_CRASHSTORAGE_BUCKET_NAME}" python ./bin/gcs_cli.py list_buckets -echo "Delete and create SQS queue..." -python ./bin/sqs_cli.py delete "${CRASHMOVER_CRASHPUBLISH_QUEUE_NAME}" -python ./bin/sqs_cli.py create "${CRASHMOVER_CRASHPUBLISH_QUEUE_NAME}" - echo "Delete and create Pub/Sub topic..." python ./bin/pubsub_cli.py delete_topic "${CRASHMOVER_CRASHPUBLISH_PROJECT_ID}" "${CRASHMOVER_CRASHPUBLISH_TOPIC_NAME}" python ./bin/pubsub_cli.py create_topic "${CRASHMOVER_CRASHPUBLISH_PROJECT_ID}" "${CRASHMOVER_CRASHPUBLISH_TOPIC_NAME}" diff --git a/bin/run_tests.sh b/bin/run_tests.sh index a3c0f409..8cb0fb3a 100755 --- a/bin/run_tests.sh +++ b/bin/run_tests.sh @@ -23,7 +23,6 @@ export PYTHONPATH=/app/:${PYTHONPATH:-} PYTEST="$(which pytest)" # Wait for services to be ready (both have the same endpoint url) -urlwait "${CRASHMOVER_CRASHPUBLISH_ENDPOINT_URL}" 15 urlwait "http://${PUBSUB_EMULATOR_HOST}" 10 urlwait "${STORAGE_EMULATOR_HOST}/storage/v1/b" 10 diff --git a/bin/s3_cli.py b/bin/s3_cli.py deleted file mode 100755 index d61350cb..00000000 --- a/bin/s3_cli.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Manipulates S3 in local dev environment. -# -# Run this in the Docker container. - -import os -import pathlib - -import boto3 -from botocore.client import ClientError, Config -import click - - -def get_client(): - session = boto3.session.Session( - aws_access_key_id=os.environ.get("CRASHMOVER_CRASHSTORAGE_ACCESS_KEY"), - aws_secret_access_key=os.environ.get( - "CRASHMOVER_CRASHSTORAGE_SECRET_ACCESS_KEY" - ), - ) - client = session.client( - service_name="s3", - config=Config(s3={"addressing_style": "path"}), - endpoint_url=os.environ.get("CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL"), - ) - return client - - -@click.group() -def s3_group(): - """Manipulate S3 in local dev environment.""" - - -@s3_group.command("create") -@click.argument("bucket") -@click.pass_context -def create_bucket(ctx, bucket): - """Create a new S3 bucket.""" - conn = get_client() - try: - conn.head_bucket(Bucket=bucket) - click.echo("Bucket %s exists." % bucket) - except ClientError: - conn.create_bucket(Bucket=bucket) - click.echo("Bucket %s created." % bucket) - - -@s3_group.command("delete") -@click.argument("bucket") -@click.pass_context -def delete_bucket(ctx, bucket): - """Delete an S3 bucket.""" - conn = get_client() - try: - conn.head_bucket(Bucket=bucket) - except ClientError: - click.echo("Bucket %s does not exist." % bucket) - return - - # Delete any objects in the bucket - resp = conn.list_objects(Bucket=bucket) - for obj in resp.get("Contents", []): - key = obj["Key"] - conn.delete_object(Bucket=bucket, Key=key) - - # Then delete the bucket - conn.delete_bucket(Bucket=bucket) - - -@s3_group.command("list_buckets") -@click.pass_context -def list_buckets(ctx): - """List S3 buckets.""" - conn = get_client() - resp = conn.list_buckets() - for bucket in resp["Buckets"]: - click.echo("%s\t%s" % (bucket["Name"], bucket["CreationDate"])) - - -@s3_group.command("list_objects") -@click.option("-d", "--details", default=False, type=bool, help="With details") -@click.argument("bucket") -@click.pass_context -def list_objects(ctx, bucket, details): - """List the contents of a bucket.""" - conn = get_client() - try: - conn.head_bucket(Bucket=bucket) - except ClientError: - click.echo("Bucket %s does not exist." % bucket) - return - - resp = conn.list_objects_v2(Bucket=bucket) - for item in resp.get("Contents", []): - if details: - click.echo(f"{item['Key']}\t{item['Size']}\t{item['LastModified']}") - else: - click.echo(f"{item['Key']}") - - -@s3_group.command("download") -@click.argument("bucket") -@click.argument("key") -@click.argument("outputdir") -@click.pass_context -def download_file(ctx, bucket, key, outputdir): - conn = get_client() - try: - conn.head_bucket(Bucket=bucket) - except ClientError: - click.echo(f"Bucket {bucket} does not exist.") - return - - keypath = pathlib.Path(outputdir) / pathlib.Path(key) - if not keypath.parent.exists(): - keypath.parent.mkdir(parents=True) - - if not keypath.parent.is_dir(): - click.echo(f"{keypath.parent} exists but is not a directory") - - with keypath.open("wb") as fp: - conn.download_fileobj(bucket, key, fp) - - -if __name__ == "__main__": - s3_group() diff --git a/bin/sqs_cli.py b/bin/sqs_cli.py deleted file mode 100755 index ab8ab6c9..00000000 --- a/bin/sqs_cli.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# SQS manipulation script. -# -# Note: Run this in the base container which has access to SQS. -# -# Usage: ./bin/sqs_cli.py [SUBCOMMAND] - -import os -import time - -import boto3 -import click - - -VISIBILITY_TIMEOUT = 2 - - -class InvalidQueueName(Exception): - """Denotes an invalid queue name.""" - - -def validate_queue_name(queue_name): - if len(queue_name) > 80: - raise InvalidQueueName("queue name is too long.") - - for c in queue_name: - if not c.isalnum() and c not in "-_": - raise InvalidQueueName("%r is not an alphanumeric, - or _ character." % c) - - -def get_client(): - session = boto3.session.Session( - aws_access_key_id=os.environ.get("CRASHMOVER_CRASHPUBLISH_ACCESS_KEY"), - aws_secret_access_key=os.environ.get( - "CRASHMOVER_CRASHPUBLISH_SECRET_ACCESS_KEY" - ), - ) - client = session.client( - service_name="sqs", - region_name=os.environ.get("CRASHMOVER_CRASHPUBLISH_REGION"), - endpoint_url=os.environ.get("CRASHMOVER_CRASHPUBLISH_ENDPOINT_URL"), - ) - return client - - -@click.group() -def sqs_group(): - """Local dev environment SQS manipulation script.""" - - -@sqs_group.command("list_messages") -@click.argument("queue") -@click.pass_context -def list_messages(ctx, queue): - """List messages in queue.""" - conn = get_client() - try: - resp = conn.get_queue_url(QueueName=queue) - queue_url = resp["QueueUrl"] - except conn.exceptions.QueueDoesNotExist: - click.echo("Queue %s does not exist.") - return - - # NOTE(willkg): Since the VisibilityTimeout is set to VISIBILITY_TIMEOUT and - # messages aren't deleted, items aren't pulled out of the queue permanently. - # However, if you run list_messages twice in rapid succession, VisibilityTimeout may - # not have passed, so we wait the timeout amount first. - time.sleep(VISIBILITY_TIMEOUT) - - is_empty = True - while True: - resp = conn.receive_message( - QueueUrl=queue_url, - WaitTimeSeconds=0, - VisibilityTimeout=VISIBILITY_TIMEOUT, - ) - msgs = resp.get("Messages", []) - if not msgs: - break - - is_empty = False - for msg in msgs: - click.echo("%s" % msg["Body"]) - - if is_empty: - click.echo("Queue %s is empty." % queue) - - -@sqs_group.command("send_message") -@click.argument("queue") -@click.argument("message") -@click.pass_context -def send_message(ctx, queue, message): - """Add a message to a queue.""" - conn = get_client() - try: - resp = conn.get_queue_url(QueueName=queue) - queue_url = resp["QueueUrl"] - except conn.exceptions.QueueDoesNotExist: - click.echo("Queue %s does not exist.") - return - - conn.send_message(QueueUrl=queue_url, MessageBody=message) - click.echo("Message sent.") - - -@sqs_group.command("list_queues") -@click.pass_context -def list_queues(ctx): - """List queues.""" - conn = get_client() - resp = conn.list_queues() - - for queue_url in resp.get("QueueUrls", []): - queue_name = queue_url.rsplit("/", 1)[1] - click.echo(queue_name) - - -@sqs_group.command("create") -@click.argument("queue") -@click.pass_context -def create(ctx, queue): - """Create SQS queue.""" - queue = queue.strip() - if not queue: - click.echo("Queue name required.") - return - - conn = get_client() - validate_queue_name(queue) - try: - conn.get_queue_url(QueueName=queue) - click.echo("Queue %s already exists." % queue) - return - except conn.exceptions.QueueDoesNotExist: - pass - conn.create_queue(QueueName=queue) - click.echo("Queue %s created." % queue) - - -@sqs_group.command("delete") -@click.argument("queue") -@click.pass_context -def delete(ctx, queue): - """Delete SQS queue.""" - queue = queue.strip() - if not queue: - click.echo("Queue name required.") - return - - conn = get_client() - try: - resp = conn.get_queue_url(QueueName=queue) - except conn.exceptions.QueueDoesNotExist: - click.echo("Queue %s does not exist." % queue) - return - - queue_url = resp["QueueUrl"] - conn.delete_queue(QueueUrl=queue_url) - click.echo("Queue %s deleted." % queue) - - -if __name__ == "__main__": - sqs_group() diff --git a/docker-compose.yml b/docker-compose.yml index a4629058..ca6e5bfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,6 @@ services: links: - fakesentry - gcs-emulator - - localstack - pubsub - statsd volumes: @@ -49,7 +48,6 @@ services: links: - fakesentry - gcs-emulator - - localstack - pubsub - statsd volumes: @@ -65,7 +63,6 @@ services: links: - fakesentry - gcs-emulator - - localstack - pubsub - statsd @@ -81,7 +78,6 @@ services: links: - fakesentry - gcs-emulator - - localstack - pubsub - statsd @@ -98,7 +94,6 @@ services: links: - fakesentry - gcs-emulator - - localstack - pubsub - statsd @@ -117,18 +112,6 @@ services: ports: - "${EXPOSE_PUBSUB_EMULATOR_PORT:-5010}:5010" - # https://hub.docker.com/r/localstack/localstack/ - # localstack running a fake S3 and SQS - localstack: - image: localstack/localstack:3.4.0 - environment: - - SERVICES=s3,sqs - - DEFAULT_REGION=us-east-1 - - HOSTNAME=localstack - - HOSTNAME_EXTERNAL=localstack - ports: - - "${EXPOSE_LOCALSTACK_PORT:-4566}:4566" - # https://hub.docker.com/r/kamon/grafana_graphite/ # username: admin, password: admin statsd: diff --git a/docker/config/local_dev.env b/docker/config/local_dev.env index 05cfce66..4664944f 100644 --- a/docker/config/local_dev.env +++ b/docker/config/local_dev.env @@ -1,4 +1,4 @@ -# prod-like Antenna environment with localstack. +# prod-like Antenna environment with local emulators. # # See https://antenna.readthedocs.io/ for documentation. @@ -13,8 +13,8 @@ STATSD_HOST=statsd STATSD_NAMESPACE=mcboatface # Crashmover settings -CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.s3.crashstorage.S3CrashStorage -CRASHMOVER_CRASHPUBLISH_CLASS=antenna.ext.sqs.crashpublish.SQSCrashPublish +CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage +CRASHMOVER_CRASHPUBLISH_CLASS=antenna.ext.pubsub.crashpublish.PubSubCrashPublish # Pub/Sub settings CRASHMOVER_CRASHPUBLISH_PROJECT_ID=local-dev-socorro @@ -27,20 +27,8 @@ PUBSUB_EMULATOR_HOST=pubsub:5010 # Set GCS library to use emulator STORAGE_EMULATOR_HOST=http://gcs-emulator:8001 -# S3 settings -CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL=http://localstack:4566 -CRASHMOVER_CRASHSTORAGE_REGION=us-east-1 -CRASHMOVER_CRASHSTORAGE_ACCESS_KEY=foo -CRASHMOVER_CRASHSTORAGE_SECRET_ACCESS_KEY=foo -# Used for S3 and GCS +# GCS settings CRASHMOVER_CRASHSTORAGE_BUCKET_NAME=antennabucket -# SQS settings -CRASHMOVER_CRASHPUBLISH_ENDPOINT_URL=http://localstack:4566 -CRASHMOVER_CRASHPUBLISH_REGION=us-east-1 -CRASHMOVER_CRASHPUBLISH_ACCESS_KEY=foo -CRASHMOVER_CRASHPUBLISH_SECRET_ACCESS_KEY=foo -CRASHMOVER_CRASHPUBLISH_QUEUE_NAME=local_dev_socorro_standard - # Set up fakesentry SECRET_SENTRY_DSN=http://public@fakesentry:8090/1 diff --git a/docker/config/test.env b/docker/config/test.env index 96137fa2..ae49ed06 100644 --- a/docker/config/test.env +++ b/docker/config/test.env @@ -9,8 +9,5 @@ CRASHMOVER_CRASHPUBLISH_PROJECT_ID=test-dev-socorro CRASHMOVER_CRASHPUBLISH_TOPIC_NAME=test_dev_socorro_standard CRASHMOVER_CRASHPUBLISH_SUBSCRIPTION_NAME=test_dev_socorro_sub -# Used for S3 and GCS +# GCS settings CRASHMOVER_CRASHSTORAGE_BUCKET_NAME=testbucket - -# SQS settings -CRASHMOVER_CRASHPUBLISH_QUEUE_NAME=test_dev_socorro_standard diff --git a/docs/configuration.rst b/docs/configuration.rst index a939bbc5..dc290c01 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -17,9 +17,7 @@ when you run Antenna using ``docker compose``. In a server environment, configuration is pulled in from the process environment. -Here's an example. This uses Datadog installed on the EC2 node for metrics and -also IAM bound to the EC2 node that Antenna is running on so it doesn't need S3 -credentials for crashstorage. +Here's an example. This uses statsd with datadog extensions installed on localhost for metrics. :: @@ -27,9 +25,9 @@ credentials for crashstorage. STATSD_NAMESPACE=mcboatface # BreakdpadSubmitterResource settings - CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.s3.crashstorage.S3CrashStorage + CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage - # S3CrashStorage and S3Connection settings + # GcsCrashStorage settings CRASHMOVER_CRASHSTORAGE_BUCKET_NAME=org-myorg-mybucket @@ -189,42 +187,6 @@ and implement that. CRASHMOVER_CRASHSTORAGE_FS_ROOT=/tmp/whatever -AWS S3 ------- - -The ``S3CrashStorage`` class will save crash data to AWS S3. You might be able -to use this to save to other S3-like systems, but that's not tested or -supported. - -.. autocomponentconfig:: antenna.ext.s3.connection.S3Connection - :show-docstring: - :case: upper - :namespace: crashmover_crashstorage - :show-table: - - When set as the CrashMover crashstorage class, configuration - for this class is in the ``CRASHMOVER_CRASHSTORAGE`` namespace. - - Example:: - - CRASHMOVER_CRASHSTORAGE_BUCKET_NAME=mybucket - CRASHMOVER_CRASHSTORAGE_REGION=us-west-2 - CRASHMOVER_CRASHSTORAGE_ACCESS_KEY=somethingsomething - CRASHMOVER_CRASHSTORAGE_SECRET_ACCESS_KEY=somethingsomething - - -.. autocomponentconfig:: antenna.ext.s3.crashstorage.S3CrashStorage - :show-docstring: - :case: upper - :namespace: crashmover_crashstorage - - When set as the CrashMover crashstorage class, configuration - for this class is in the ``CRASHMOVER_CRASHSTORAGE`` namespace. - - Generally, if the default connection class is fine, you don't need to do any - configuration here. - - Google Cloud Storage -------------------- @@ -276,18 +238,3 @@ topic. for this class is in the ``CRASHMOVER_CRASHPUBLISH`` namespace. You need to set the project id and topic name. - - -AWS SQS -------- - -The ``SQSCrashPublish`` class will publish crash ids to an AWS SQS queue. - -.. autocomponentconfig:: antenna.ext.sqs.crashpublish.SQSCrashPublish - :show-docstring: - :case: upper - :namespace: crashmover_crashpublish - :show-table: - - When set as the CrashMover crashpublish class, configuration - for this class is in the ``CRASHMOVER_CRASHPUBLISH`` namespace. diff --git a/docs/deploy.rst b/docs/deploy.rst index b73e32ca..1b50d007 100644 --- a/docs/deploy.rst +++ b/docs/deploy.rst @@ -32,7 +32,7 @@ Antenna exposes several URL endpoints to help you run it at scale. ``/__heartbeat__`` This endpoint returns some more data. Depending on how you have Antenna - configured, this might do a HEAD on the s3 bucket or other things. It + configured, this might do a HEAD on the GCS bucket or other things. It returns its findings in the HTTP response body. ``/__version__`` @@ -64,11 +64,11 @@ Environments Will this run in Heroku? Probably, but you'll need to do some Heroku footwork to set it up. -Will this run on AWS? Yes--that's what we do. +Will this run on GCP? Yes--that's what we do. Will this run on [insert favorite environment]? I have no experience with other systems, but it's probably the case you can get it to work. If you can't save -crashes to Amazon S3, you can always write your own storage class to save it +crashes to Google Cloud Storage, you can always write your own storage class to save it somewhere else. @@ -96,11 +96,11 @@ What happens after Antenna collects a crash? ============================================ Antenna saves the crash to the crash storage system you specify. We save our -crashes to AWS S3. +crashes to Google Cloud Storage. Then it publishes the crash to the designated crash queue. We queue crashes for -processing with AWS SQS. The processor pulls crash report ids to process from -the AWS SQS queue. +processing with Google Cloud Pub/Sub. The processor pulls crash report ids to +process from Pub/Sub. Troubleshooting @@ -122,28 +122,28 @@ Things to check: 3. Is the configuration correct? -AWS S3 bucket permission issues +Google Cloud Storage bucket permission issues ------------------------------- -At startup, Antenna will try to Head the AWS S3 bucket and if it fails, will -refuse to start up. It does this so that it doesn't start up, then get a crash -and then fail to submit the crash due to permission issues. At that point, you'd -have lost the crash. +At startup, Antenna will try to write to the Google Cloud Storage bucket and if it +fails, will refuse to start up. It does this so that it doesn't start up, then +get a crash and then fail to submit the crash due to permission issues. At that +point, you'd have lost the crash. If you're seeing errors like:: - [ERROR] antenna.app: Unhandled startup exception: ... botocore.exceptions.ClientError: - An error occurred (403) when calling the HeadBucket operation: Forbidden + 2024-07-15 23:15:30,532 DEBUG - antenna - antenna.app - Verifying GcsCrashStorage.verify_write_to_bucket + [2024-07-15 23:15:30 +0000] [9] [ERROR] Worker (pid:10) exited with code 3 it means that the credentials that Antenna is using don't have the right -permissions to the AWS S3 bucket. +permissions to the Google Cloud Storage bucket. Things to check: 1. Check the bucket and region that Antenna is configured with. It'll be in the logs when Antenna starts up. -2. Check that Antenna has the right AWS credentials. +2. Check that Antenna has the right GCP credentials. 3. Try using the credentials that Antenna is using to access the bucket. diff --git a/docs/dev.rst b/docs/dev.rst index f3314f5c..31791cd2 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -48,7 +48,7 @@ production, see documentation_. Anytime you want to update the containers, you can run ``make build``. -5. Set up local SQS and S3 services: +5. Set up local Pub/Sub and GCS services: .. code-block:: shell @@ -79,8 +79,8 @@ production, see documentation_. web_1 | [2022-09-13 14:21:45 +0000] [8] [INFO] Using worker: sync web_1 | [2022-09-13 14:21:45 +0000] [9] [INFO] Booting worker with pid: 9 web_1 | 2022-09-13 14:21:45,461 INFO - antenna - antenna.liblogging - set up logging logging_level=DEBUG debug=True host_id=097fa14aec1e processname=antenna - web_1 | 2022-09-13 14:21:45,573 DEBUG - antenna - antenna.app - registered S3CrashStorage.verify_write_to_bucket for verification - web_1 | 2022-09-13 14:21:45,612 DEBUG - antenna - antenna.app - registered SQSCrashPublish.verify_queue for verification + web_1 | 2022-09-13 14:21:45,573 DEBUG - antenna - antenna.app - registered GcsCrashStorage.verify_write_to_bucket for verification + web_1 | 2022-09-13 14:21:45,612 DEBUG - antenna - antenna.app - registered PubSubCrashPublish.verify_topic for verification web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - BASEDIR=/app web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - LOGGING_LEVEL=DEBUG web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - LOCAL_DEV_ENV=True @@ -90,26 +90,19 @@ production, see documentation_. web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - SECRET_SENTRY_DSN=***** web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - HOST_ID= web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CONCURRENT_CRASHMOVERS=8 - web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.s3.crashstorage.S3CrashStorage - web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_CLASS=antenna.ext.sqs.crashpublish.SQSCrashPublish - web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_CONNECTION_CLASS=antenna.ext.s3.connection.S3Connection - web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_ACCESS_KEY=foo - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_SECRET_ACCESS_KEY=***** - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_REGION=us-east-1 - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL=http://localstack:4566 + web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage + web_1 | 2022-09-13 14:21:45,613 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_CLASS=antenna.ext.pubsub.crashpublish.PubSubCrashPublish web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHSTORAGE_BUCKET_NAME=antennabucket - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_ACCESS_KEY=foo - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_SECRET_ACCESS_KEY=***** - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_REGION=us-east-1 - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_ENDPOINT_URL=http://localstack:4566 - web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_QUEUE_NAME=local_dev_socorro_standard + web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_PROJECT_ID=local-dev-socorro + web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_TOPIC_NAME=local_dev_socorro_standard + web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - CRASHMOVER_CRASHPUBLISH_TIMEOUT=5 web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - BREAKPAD_DUMP_FIELD=upload_file_minidump web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - BREAKPAD_THROTTLER_RULES=antenna.throttler.MOZILLA_RULES web_1 | 2022-09-13 14:21:45,614 INFO - antenna - antenna.app - BREAKPAD_THROTTLER_PRODUCTS=antenna.throttler.MOZILLA_PRODUCTS web_1 | 2022-09-13 14:21:45,661 INFO - antenna - markus.backends.datadog - DatadogMetrics configured: statsd:8125 mcboatface web_1 | 2022-09-13 14:21:45,668 DEBUG - antenna - antenna.app - Verification starting. - web_1 | 2022-09-13 14:21:45,669 DEBUG - antenna - antenna.app - Verifying SQSCrashPublish.verify_queue - web_1 | 2022-09-13 14:21:45,678 DEBUG - antenna - antenna.app - Verifying S3CrashStorage.verify_write_to_bucket + web_1 | 2022-09-13 14:21:45,669 DEBUG - antenna - antenna.app - Verifying PubSubCrashPublish.verify_topic + web_1 | 2022-09-13 14:21:45,678 DEBUG - antenna - antenna.app - Verifying GcsCrashStorage.verify_write_to_bucket web_1 | 2022-09-13 14:21:45,699 DEBUG - antenna - antenna.app - Verification complete: everything is good! web_1 | 2022-09-13 14:21:45,699 INFO - antenna - antenna.app - Antenna is running! http://localhost:8000/ web_1 | 2022-09-13 14:21:45,700 INFO - antenna - markus - METRICS|2022-09-13 14:21:45|gauge|crashmover.work_queue_size|0| @@ -122,7 +115,7 @@ production, see documentation_. $ docker compose ps - You should see containers with names ``web``, ``statsd`` and ``localstack``. + You should see containers with names ``web``, ``statsd``, ``pubsub`` and ``gcs-emulator``. 3. Send in a crash report: @@ -155,18 +148,12 @@ production, see documentation_. web_1 | 2022-09-13 14:23:19,374 INFO - antenna - markus - METRICS|2022-09-13 14:23:19|incr|crashmover.save_crash.count|1| web_1 | 2022-09-13 14:23:22,814 INFO - antenna - markus - METRICS|2022-09-13 14:23:22|gauge|crashmover.work_queue_size|0| - 4. See the data in localstack/gcs-emulator: + 4. See the data in gcs-emulator: - The ``localstack`` container stores data in memory and the data doesn't + The ``gcs-emulator`` container stores data in memory and the data doesn't persist between container restarts. - You can use the ``bin/s3_cli.py`` to access it: - - .. code-block:: shell - - $ docker compose run --rm web shell python bin/s3_cli.py list_buckets - - For gcs-emulator you can use ``bin/gcs_cli.py`` to access it: + You can use the ``bin/gcs_cli.py`` to access it: .. code-block:: shell diff --git a/docs/overview.rst b/docs/overview.rst index 01232310..672d13b6 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -35,13 +35,14 @@ Antenna is built with the following requirements: 1. **Return a crash id to the client quickly** Antenna should return a crash id and close the HTTP connection as quickly as - possible. This means we need to save to AWS S3 as a separate step. + possible. This means we need to save to Google Cloud Storage as a separate + step. 2. **Try hard not to drop crashes** Antenna tries hard not to drop crashes and lose data. It tries to get the - crash to AWS S3 as quickly as possible so that it's sitting on as few crash - reports as possible. + crash to Google Cloud Storage as quickly as possible so that it's sitting on + as few crash reports as possible. 3. **Minimal dependencies** @@ -96,11 +97,11 @@ This is the rough data flow: 3. Then ``BreakpadSubmitterResource`` passes the data to the crashmover to save and publish. - If crashstorage is ``S3CrashStorage``, then the crashmover saves the crash - report data to AWS S3. + If crashstorage is ``GcsCrashStorage``, then the crashmover saves the crash + report data to Google Cloud Storage. If the save is successful, then the crashmover publishes the crash report - id to the AWS SQS standard queue for processing. + id to the Google Cloud Pub/Sub standard queue topic for processing. At this point, the HTTP POST has been handled, the crash id is sent to the crash reporter client and the HTTP connection ends. @@ -194,12 +195,12 @@ Here are some good ones: * ``breakpad_resource.crash_save.time`` - Timing. This is the time it took to save the crash to S3. + Timing. This is the time it took to save the crash to Google Cloud Storage. * ``breakpad_resource.crash_handling.time`` Timing. This is the total time the crash was in Antenna-land from receiving - the crash to saving it to S3. + the crash to saving it to Google Cloud Storage. Sentry @@ -214,8 +215,8 @@ instance--either will work fine. Cloud storage file hierarchy --------------------- -If you use the Amazon Web Services S3 or Google Cloud Storage crashstorage -component, then crashes get saved in this hierarchy in the bucket: +If you use the Google Cloud Storage crashstorage component, then crashes get +saved in this hierarchy in the bucket: * ``/v1/raw_crash//`` * ``/v1/dump_names/`` diff --git a/pyproject.toml b/pyproject.toml index a382060e..ba61f911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,15 +18,10 @@ docstring-quotes = "double" [tool.pytest.ini_options] -addopts = "-rsxX --tb=native --showlocals -m 'not gcp'" +addopts = "-rsxX --tb=native --showlocals" norecursedirs = [".git", "docs", "bin"] testpaths = "tests/" -markers = [ - "aws: tests that require aws backends to be configured in the environment. this is the default.", - "gcp: tests that require gcp backends to be configured in the environment. skipped unless explicitly requested.", -] - filterwarnings = [ "error", # Falcon currently uses cgi which is going away in python 3.13 diff --git a/requirements.in b/requirements.in index 879ef15c..0ed18890 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,5 @@ # Production requirements attrs==23.2.0 -boto3==1.34.95 -botocore==1.34.95 datadog==0.49.1 dockerflow==2024.4.2 everett==3.3.0 diff --git a/requirements.txt b/requirements.txt index df27579a..8e8686e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,17 +22,6 @@ bandit==1.7.9 \ --hash=sha256:52077cb339000f337fb25f7e045995c4ad01511e716e5daac37014b9752de8ec \ --hash=sha256:7c395a436743018f7be0a4cbb0a4ea9b902b6d87264ddecf8cfdc73b4f78ff61 # via -r requirements.in -boto3==1.34.95 \ - --hash=sha256:decf52f8d5d8a1b10c9ff2a0e96ee207ed79e33d2e53fdf0880a5cbef70785e0 \ - --hash=sha256:e836b71d79671270fccac0a4d4c8ec239a6b82ea47c399b64675aa597d0ee63b - # via -r requirements.in -botocore==1.34.95 \ - --hash=sha256:6bd76a2eadb42b91fa3528392e981ad5b4dfdee3968fa5b904278acf6cbf15ff \ - --hash=sha256:ead5823e0dd6751ece5498cb979fd9abf190e691c8833bcac6876fd6ca261fa7 - # via - # -r requirements.in - # boto3 - # s3transfer build==1.0.3 \ --hash=sha256:538aab1b64f9828977f84bc63ae570b060a8ed1be419e7870b8b4fc5e6ea553b \ --hash=sha256:589bf99a67df7c9cf07ec0ac0e5e2ea5d4b37ac63301c4986d1acb126aa83f8f @@ -411,12 +400,6 @@ jinja2==3.1.4 \ --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d # via sphinx -jmespath==1.0.1 \ - --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ - --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe - # via - # boto3 - # botocore markdown-it-py==3.0.0 \ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb @@ -569,9 +552,7 @@ pytest==8.2.1 \ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 - # via - # botocore - # freezegun + # via freezegun pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ @@ -661,10 +642,6 @@ ruff==0.5.0 \ --hash=sha256:ed5c4df5c1fb4518abcb57725b576659542bdbe93366f4f329e8f398c4b71178 \ --hash=sha256:ee770ea8ab38918f34e7560a597cc0a8c9a193aaa01bfbd879ef43cb06bd9c4c # via -r requirements.in -s3transfer==0.10.1 \ - --hash=sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19 \ - --hash=sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d - # via boto3 sentry-sdk==2.7.1 \ --hash=sha256:25006c7e68b75aaa5e6b9c6a420ece22e8d7daec4b7a906ffd3a8607b67c037b \ --hash=sha256:ef1b3d54eb715825657cd4bb3cb42bb4dc85087bac14c56b0fd8c21abd968c9a @@ -733,7 +710,6 @@ urllib3==2.2.2 \ --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via - # botocore # requests # sentry-sdk urlwait==1.0 \ diff --git a/systemtest/README.rst b/systemtest/README.rst index d7dda1b3..4a0e7c38 100644 --- a/systemtest/README.rst +++ b/systemtest/README.rst @@ -21,12 +21,10 @@ In a terminal, run:: app@xxx:/app$ ./systemtest/test_env.sh ENV -Additional arguments will be passed through to pytest. For example, if testing gcp backends, use -pytest markers to select the tests for the configured cloud provider:: +Additional arguments will be passed through to pytest. For example:: $ make shell - app@xxx:/app$ ./systemtest/test_env.sh ENV -m gcp # only gcp - app@xxx:/app$ ./systemtest/test_env.sh ENV -m 'not aws' # gcp and unmarked + app@xxx:/app$ ./systemtest/test_env.sh ENV -- test_dockerflow.py If you're running against ``local``, you'll need to be running antenna @@ -54,7 +52,7 @@ Rules of systemtest if not nginx: pytest.skip("test requires nginx") -3. Tests can check S3 to see if a file exists by listing objects, but +3. Tests can check GCS to see if a file exists by listing objects, but cannot get the file. 4. Tests won't check Pub/Sub at all unless they're using the Pub/Sub diff --git a/systemtest/conftest.py b/systemtest/conftest.py index ae91dcba..81120de6 100644 --- a/systemtest/conftest.py +++ b/systemtest/conftest.py @@ -7,8 +7,6 @@ import sys import os -import boto3 -from botocore.client import Config from everett.manager import ConfigManager, ConfigOSEnv from google.cloud import pubsub_v1 import pytest @@ -56,21 +54,6 @@ def posturl(config): return config("host", default="http://web:8000/").rstrip("/") + "/submit" -@pytest.fixture( - # define these params once and reference this fixture to prevent undesirable - # combinations, i.e. tests marked with aws *and* gcp for pubsub+s3 or sqs+gcs - params=[ - # tests that require a specific cloud provider backend must be marked with that - # provider, and non-default backends must be excluded by default, for example - # via pytest.ini's addopts, i.e. addopts = -m 'not gcp' - pytest.param("aws", marks=pytest.mark.aws), - pytest.param("gcp", marks=pytest.mark.gcp), - ] -) -def cloud_provider(request): - return request.param - - class GcsHelper: def __init__(self, bucket): self.bucket = bucket @@ -103,84 +86,10 @@ def list_objects(self, prefix): return [blob.name for blob in list(bucket.list_blobs(prefix=prefix))] -class S3Helper: - def __init__(self, access_key, secret_access_key, endpoint_url, region, bucket): - self.access_key = access_key - self.secret_access_key = secret_access_key - self.endpoint_url = endpoint_url - self.region = region - self.bucket = bucket - - self.logger = logging.getLogger(__name__ + ".s3_helper") - self.conn = self.connect() - - def get_config(self): - return { - "helper": "s3", - "endpoint_url": self.endpoint_url, - "region": self.region, - "bucket": self.bucket, - } - - def connect(self): - session_kwargs = {} - if self.access_key and self.secret_access_key: - session_kwargs["aws_access_key_id"] = self.access_key - session_kwargs["aws_secret_access_key"] = self.secret_access_key - - session = boto3.session.Session(**session_kwargs) - - client_kwargs = { - "service_name": "s3", - "region_name": self.region, - "config": Config(s3={"addression_style": "path"}), - } - if self.endpoint_url: - client_kwargs["endpoint_url"] = self.endpoint_url - - client = session.client(**client_kwargs) - return client - - def dump_key(self, crash_id, name): - if name in (None, "", "upload_file_minidump"): - name = "dump" - - return f"v1/{name}/{crash_id}" - - def list_objects(self, prefix): - """Return list of keys in S3 bucket.""" - self.logger.info('listing "%s" for prefix "%s"', self.bucket, prefix) - resp = self.conn.list_objects( - Bucket=self.bucket, Prefix=prefix, RequestPayer="requester" - ) - return [obj["Key"] for obj in resp["Contents"]] - - @pytest.fixture -def storage_helper(config, cloud_provider): +def storage_helper(config): """Generate and return a storage helper using env config.""" - actual_backend = "s3" - if ( - config("crashmover_crashstorage_class") - == "antenna.ext.gcs.crashstorage.GcsCrashStorage" - ): - actual_backend = "gcs" - - expect_backend = "gcs" if cloud_provider == "gcp" else "s3" - if actual_backend != expect_backend: - pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - - if actual_backend == "gcs": - return GcsHelper( - bucket=config("crashmover_crashstorage_bucket_name"), - ) - return S3Helper( - access_key=config("crashmover_crashstorage_access_key", default=""), - secret_access_key=config( - "crashmover_crashstorage_secret_access_key", default="" - ), - endpoint_url=config("crashmover_crashstorage_endpoint_url", default=""), - region=config("crashmover_crashstorage_region", default="us-west-2"), + return GcsHelper( bucket=config("crashmover_crashstorage_bucket_name"), ) @@ -224,89 +133,15 @@ def list_crashids(self): return crashids -class SQSHelper: - def __init__(self, access_key, secret_access_key, endpoint_url, region, queue_name): - self.access_key = access_key - self.secret_access_key = secret_access_key - self.endpoint_url = endpoint_url - self.region = region - self.queue_name = queue_name - self.client = self.connect() - - def connect(self): - session_kwargs = {} - if self.access_key and self.secret_access_key: - session_kwargs["aws_access_key_id"] = self.access_key - session_kwargs["aws_secret_access_key"] = self.secret_access_key - - session = boto3.session.Session(**session_kwargs) - - client_kwargs = { - "service_name": "sqs", - "region_name": self.region, - } - if self.endpoint_url: - client_kwargs["endpoint_url"] = self.endpoint_url - - client = session.client(**client_kwargs) - return client - - def list_crashids(self): - """Return crash ids in the SQS queue.""" - queue_url = self.client.get_queue_url(QueueName=self.queue_name)["QueueUrl"] - - crashids = [] - while True: - resp = self.client.receive_message( - QueueUrl=queue_url, - WaitTimeSeconds=0, - VisibilityTimeout=2, - ) - msgs = resp.get("Messages", []) - if not msgs: - break - - for msg in msgs: - data = msg["Body"] - handle = msg["ReceiptHandle"] - if data != "test": - crashids.append(data) - - self.client.delete_message(QueueUrl=queue_url, ReceiptHandle=handle) - - return crashids - - @pytest.fixture -def queue_helper(config, cloud_provider): +def queue_helper(config): """Generate and return a queue helper using env config.""" - actual_backend = "sqs" - if ( - config("crashmover_crashpublish_class") - == "antenna.ext.pubsub.crashpublish.PubSubCrashPublish" - ): - actual_backend = "pubsub" - - expect_backend = "pubsub" if cloud_provider == "gcp" else "sqs" - if actual_backend != expect_backend: - pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - - if actual_backend == "pubsub": - return PubSubHelper( - project_id=config("crashmover_crashpublish_project_id", default=""), - topic_name=config("crashmover_crashpublish_topic_name", default=""), - subscription_name=config( - "crashmover_crashpublish_subscription_name", default="" - ), - ) - return SQSHelper( - access_key=config("crashmover_crashpublish_access_key", default=""), - secret_access_key=config( - "crashmover_crashpublish_secret_access_key", default="" + return PubSubHelper( + project_id=config("crashmover_crashpublish_project_id", default=""), + topic_name=config("crashmover_crashpublish_topic_name", default=""), + subscription_name=config( + "crashmover_crashpublish_subscription_name", default="" ), - endpoint_url=config("crashmover_crashpublish_endpoint_url", default=""), - region=config("crashmover_crashpublish_region", default=""), - queue_name=config("crashmover_crashpublish_queue_name", default=""), ) diff --git a/systemtest/test_content_length.py b/systemtest/test_content_length.py index c0b38c32..e98d4744 100644 --- a/systemtest/test_content_length.py +++ b/systemtest/test_content_length.py @@ -109,12 +109,10 @@ def test_content_length_1000(self, posturl, crash_generator, nginx): assert status_code in ( # without LB, nginx drops the connection None, - # with LB in GCP if nginx drops the connection we get 502, or if LB - # times out first we get 408. clients might not retry a 4XX error - # like they will a 5XX, so we don't accept a 408 here. - 502, - # with LB in AWS if nginx drops the connection we get 504 - 504, + # with GCLB if nginx drops the connection we get 502, or if GCLB + # times out first we get 408. Only accept 408 here because we don't + # want clients that can't send a body to trigger 5xx alerts. + 408, ) def test_content_length_non_int(self, posturl, crash_generator): diff --git a/systemtest/test_dockerflow.py b/systemtest/test_dockerflow.py index 8e9d4b7f..0aae0661 100644 --- a/systemtest/test_dockerflow.py +++ b/systemtest/test_dockerflow.py @@ -12,11 +12,10 @@ def test_version(self, baseurl): data = resp.json() assert isinstance(data, dict) data_keys = list(sorted(data.keys())) - # It's got "cloud" in the local dev environment and 5 keys in a server + # It's got nothing in the local dev environment and 4 keys in a server # environment - assert data_keys == ["cloud"] or data_keys == [ + assert data_keys == [] or data_keys == [ "build", - "cloud", "commit", "source", "version", diff --git a/systemtest/test_env.sh b/systemtest/test_env.sh index a13ca53b..5067ac6f 100755 --- a/systemtest/test_env.sh +++ b/systemtest/test_env.sh @@ -25,17 +25,12 @@ case $1 in "local") # Whether or not we're running behind nginx and to run nginx tests export NGINX_TESTS=0 - # Whether or not we can verify the file was saved (need access to S3) + # Whether or not we can verify the file was saved (need access to GCS) export POST_CHECK=1 # The host to submit to export HOST=http://web:8000/ ;; - "stage" | "aws-stage") - export NGINX_TESTS=1 - export POST_CHECK=0 - export HOST=https://crash-reports.allizom.org/ - ;; - "gcp-stage") + "stage" | "gcp-stage") export NGINX_TESTS=1 export POST_CHECK=0 export HOST=https://antenna-stage.socorro.nonprod.webservices.mozgcp.net/ diff --git a/systemtest/test_post_crash.py b/systemtest/test_post_crash.py index 52c195f9..b6f7a3d6 100644 --- a/systemtest/test_post_crash.py +++ b/systemtest/test_post_crash.py @@ -42,7 +42,7 @@ def test_regular( crash_verifier, postcheck, ): - """Post a valid crash and verify the contents made it to S3.""" + """Post a valid crash and verify the contents made it to storage.""" if not postcheck: pytest.skip("no access to storage") @@ -70,7 +70,7 @@ def test_compressed_crash( crash_verifier, postcheck, ): - """Post a compressed crash and verify contents made it to S3.""" + """Post a compressed crash and verify contents made it to storage.""" if not postcheck: pytest.skip("no access to storage") diff --git a/testlib/mini_poster.py b/testlib/mini_poster.py index 5d4cebee..e1a7160c 100644 --- a/testlib/mini_poster.py +++ b/testlib/mini_poster.py @@ -289,7 +289,7 @@ def cmdline(args): elif "v1/" in parsed.raw_crash: # If there's a 'v1' in the raw_crash filename, then it's probably the case that - # willkg wants all the pieces for a crash he pulled from S3. We like willkg, so + # willkg wants all the pieces for a crash he pulled from GCS. We like willkg, so # we'll help him out by doing the legwork. raw_crash_path = Path(parsed.raw_crash) if str(raw_crash_path.parents[2]).endswith("v1"): diff --git a/testlib/s3mock.py b/testlib/s3mock.py deleted file mode 100644 index 982b2d4e..00000000 --- a/testlib/s3mock.py +++ /dev/null @@ -1,352 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -"""This module holds bits to make it easier to test Antenna and related scripts. - -This contains ``S3Mock`` which is the mocking system we use for writing tests -that enforce HTTP conversations. - -It's in this module because at some point, it might make sense to extract this -into a separate library. - -""" - -import dataclasses -import io -import unittest - -from urllib3.response import HTTPResponse - - -class ThouShaltNotPass(Exception): - """Raised when an unhandled HTTP request is run.""" - - pass - - -# Map of status code -> reason -CODE_TO_REASON = { - 200: "OK", - 201: "Created", - 204: "No Content", - 206: "Partial Content", - 304: "Not Modified", - 307: "Temporary Redirect", - 403: "Forbidden", - 404: "Not Found", -} - - -@dataclasses.dataclass -class Request: - method: str - url: str - body: io.BytesIO - headers: dict - scheme: str - host: str - port: int - - -class Step: - def __init__(self, method, url, body=None, resp=None): - # Used to match the request - self.method = method - self.url = url - self.body = body - - # Response - self.resp = resp - - def match(self, request): - def check_body(request_body): - if self.body is None: - return True - body = request_body.read() - request_body.seek(0) - return self.body == body - - return ( - self.method == request.method - # The url can be any one of /path, scheme://host/path, or - # scheme://host:port/path - and self.url - in ( - request.url, - "%s://%s%s" % (request.scheme, request.host, request.url), - "%s://%s:%s%s" - % (request.scheme, request.host, request.port, request.url), - ) - and check_body(request.body) - ) - - def build_response(self, request): - status_code = self.resp["status_code"] - headers = self.resp["headers"] - body = self.resp["body"] - - response = HTTPResponse( - body=io.BytesIO(body), - headers=headers, - status=status_code, - request_url=request.url, - request_method=request.method, - reason=CODE_TO_REASON[status_code], - preload_content=False, - decode_content=False, - ) - - if "content-type" not in headers: - headers["content-type"] = "text/xml" - if "content-length" not in headers: - headers["content-length"] = len(body) - - response.request = request - - return response - - -def serialize_request(request): - """Takes a request object and "serializes" it into bytes - - This can be printed and is HTTP-request-like. - - :arg request: ``botocore.awsrequest.AWSPreparedRequest`` - - :returns: bytes of serialized request - - """ - output = [] - - def ln(part): - if isinstance(part, str): - part = part.encode("utf-8") - output.append(part) - - ln("%s %s" % (request.method, request.url)) - for key, val in request.headers.items(): - ln("%s: %s" % (key, val)) - ln("") - if request.body is not None: - data = request.body.read() - request.body.seek(0) - else: - data = b"" - ln(data) - ln("") - - return b"\n".join(output) - - -def serialize_response(response): - """Takes a response object and "seralizes" it into bytes - - This can be printed and is HTTP-response-like. - - :arg response; ``urllib3.response.HTTPResponse`` - - :returns: bytes of serialized response - - """ - output = [] - - def ln(part): - if isinstance(part, str): - part = part.encode("utf-8") - output.append(part) - - ln("%s %s" % (response.status_code, response.reason)) - for key, val in response.headers.items(): - ln("%s: %s" % (key, val)) - ln("") - ln(response.content) - ln("") - - return b"\n".join(output) - - -class S3Mock: - """Provide a configurable mock for Boto3's S3 bits. - - Boto3 uses botocore which uses s3transfer which uses urllib3 to do REST API calls. - - ``S3Mock`` mocks urlopen which allows it to intercept all outgoing HTTP requests. - This lets us do two things: - - 1. assert HTTP conversations happen in a specified way in tests - 2. prevent any unexpected HTTP requests from hitting the network - - - **Usage** - - ``S3Mock`` is used as a context manager. - - Basic use:: - - with S3Mock() as s3: - # do things here - - - Enforce specific conversation flows with S3 by adding one or more - conversation steps:: - - with S3Mock() as s3: - # Match on request method and url - s3.add_step( - method='PUT', - url='http://fakes3:4569/fakebucket/some/key', - resp=s3.fake_response(status_code=200) - ) - - # Match on request method, url and body - s3.add_step( - method='PUT', - url='http://fakes3:4569/fakebucket/some/other/key', - body=b'["upload_file_minidump"]', - resp=s3.fake_response(status_code=200) - ) - - # ... do whatever here - - # Assert that the entire expected conversation has occurred - assert s3.remaining_conversation() == [] - - - In that, the first HTTP request has to be a ``PUT - http://fakes3:4569/fakebucket/some/key`` and if it's not, then an exception - is thrown with helpful information. If that's the first request, then the - ``resp`` is used to generate a response object which is returned. - - The second request has to have the specified method, url and body. If not, - then an exception is thrown. - - You can specify a method, url and body to match the request. - - ``S3Mock`` has a ``fake_response`` method that will generate a fake response - to return when the request matches. - - After all the steps and other things that you want to do are done, then you - can assert that the entire expected conversation has occurred. - - - **Troubleshooting** - - If ``S3Mock`` is thwarting you, you can tell it to ``run_on_error`` and it'll - execute the urlopen and tell you want the response was. It'll also tell you - what it expected. This helps debugging assertions on HTTP conversations. - - Usage:: - - with S3Mock() as s3: - s3.run_on_error() - # ... add steps, etc - - """ - - def __init__(self): - # The expected conversation specified by the developer - self.expected_conv = [] - - # The actual conversation that happened - self.conv = [] - - self._patcher = None - - self._run_on_error = False - - def mocked_urlopen(self, pool, method, url, body=None, headers=None, **kwargs): - req = Request(method, url, body, headers, pool.scheme, pool.host, pool.port) - - if self.expected_conv and self.expected_conv[0].match(req): - step = self.expected_conv.pop(0) - resp = step.build_response(req) - self.conv.append((req, step, resp)) - return resp - - # NOTE(willkg): We use print here because fiddling with the logging - # framework inside test scaffolding is "tricky". - print("THWARTED SEND:\nHTTP Request:\n%s" % serialize_request(req)) - if self.expected_conv: - step = self.expected_conv[0] - print("Expected:\n%s %s\n%s" % (step.method, step.url, step.body)) - else: - print("Expected: nothing") - - if self._patcher and self._run_on_error: - resp = self._patcher.get_original()[0]( - pool, method=method, url=url, body=body, headers=headers, **kwargs - ) - print("HTTP Response:\n%s" % serialize_response(resp)) - - raise ThouShaltNotPass("Preventing unexpected urlopen call") - - def run_on_error(self): - """Set S3Mock to run the HTTP request if it hits a conversation error.""" - self._run_on_error = True - - def __enter__(self): - self.start_mock() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.stop_mock() - - def fake_response(self, status_code, headers=None, body=b""): - """Generates a fake response for a step in an HTTP conversation - - Example:: - - with S3Mock() as s3: - s3.add_step( - method='PUT', - url='http://fakes3:4569/...', - resp=s3.fake_response(status_code=200) - ) - - """ - if headers is None: - headers = {} - - return {"status_code": status_code, "headers": headers, "body": body} - - def add_step(self, method, url, body=None, resp=None): - """Adds a step to the expected HTTP conversation - - Generates a step which will match a request on method, url and body and - return a specified response. - - To build the response, use ``S3Mock.fake_response``. For example:: - - with S3Mock() as s3: - s3.add_step( - method='PUT', - url='http://fakes3:4569/...', - resp=s3.fake_response(status_code=200) - ) - - - :arg str method: method to match on (``GET``, ``POST``, ``PUT`` and so on) - :arg str url: the url to match - :arg bytes body: the body to match - :arg dict resp: the response to return--use ``S3Mock.fake_response`` to - build this - - """ - self.expected_conv.append(Step(method=method, url=url, body=body, resp=resp)) - - def remaining_conversation(self): - """Returns the remaining conversation to happen""" - return self.expected_conv - - def start_mock(self): - def _mocked_urlopen(pool, *args, **kwargs): - return self.mocked_urlopen(pool, *args, **kwargs) - - path = "urllib3.connectionpool.HTTPConnectionPool.urlopen" - self._patcher = unittest.mock.patch(path, _mocked_urlopen) - self._patcher.start() - - def stop_mock(self): - self._patcher.stop() - self._patcher = None diff --git a/tests/conftest.py b/tests/conftest.py index 1412af72..400bc2fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,8 +8,6 @@ import sys from unittest import mock -import boto3 -from botocore.client import ClientError as BotoClientError, Config as BotoConfig from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv from falcon.request import Request from falcon.testing.helpers import create_environ @@ -29,7 +27,6 @@ from antenna.app import get_app, setup_logging # noqa from antenna.app import reset_verify_funs # noqa -from testlib.s3mock import S3Mock # noqa class CaptureMetricsUsed(BackendBase): @@ -132,24 +129,6 @@ def test_foo(client, tmpdir): return AntennaTestClient(get_app(AntennaTestClient.build_config())) -@pytest.fixture -def s3mock(): - """Returns an s3mock context that lets you do S3-related tests - - Usage:: - - def test_something(s3mock): - s3mock.add_step( - method='PUT', - url='...' - resp=s3mock.fake_response(status_code=200) - ) - - """ - with S3Mock() as s3: - yield s3 - - @pytest.fixture def gcs_client(): if os.environ.get("STORAGE_EMULATOR_HOST"): @@ -186,49 +165,6 @@ def gcs_helper(gcs_client): gcs_client.get_bucket(bucket_name).delete(force=True) -@pytest.fixture -def s3_client(): - def get_env_var(key): - return os.environ[f"CRASHMOVER_CRASHSTORAGE_{key}"] - - session = boto3.session.Session( - aws_access_key_id=get_env_var("ACCESS_KEY"), - aws_secret_access_key=get_env_var("SECRET_ACCESS_KEY"), - ) - client = session.client( - service_name="s3", - config=BotoConfig(s3={"addressing_style": "path"}), - endpoint_url=get_env_var("ENDPOINT_URL"), - ) - return client - - -@pytest.fixture -def s3_helper(s3_client): - """Sets up bucket, yields s3_client, and tears down when test is done.""" - - def delete_bucket(s3_client, bucket_name): - resp = s3_client.list_objects(Bucket=bucket_name) - for obj in resp.get("Contents", []): - key = obj["Key"] - s3_client.delete_object(Bucket=bucket_name, Key=key) - - # Then delete the bucket - s3_client.delete_bucket(Bucket=bucket_name) - - # Set up - bucket_name = os.environ["CRASHMOVER_CRASHSTORAGE_BUCKET_NAME"] - try: - delete_bucket(s3_client, bucket_name) - except BotoClientError: - s3_client.create_bucket(Bucket=bucket_name) - - yield s3_client - - # Tear down - delete_bucket(s3_client, bucket_name) - - @pytest.fixture def metricsmock(): """Returns MetricsMock that a context to record metrics records diff --git a/tests/test_bin.py b/tests/test_bin.py index 6d7d1c3e..57131cbc 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -30,23 +30,3 @@ def test_basic(self): runner = CliRunner() result = runner.invoke(gcs_group, []) assert result.exit_code == 0 - - -class TestS3Cli: - def test_basic(self): - """Basic test to make sure s3_cli imports and runs at all.""" - from s3_cli import s3_group - - runner = CliRunner() - result = runner.invoke(s3_group, []) - assert result.exit_code == 0 - - -class TestSQSCli: - def test_basic(self): - """Basic test to make sure sqs_cli imports and runs at all.""" - from sqs_cli import sqs_group - - runner = CliRunner() - result = runner.invoke(sqs_group, []) - assert result.exit_code == 0 diff --git a/tests/test_health_resource.py b/tests/test_health_resource.py index c958c05d..ae425f94 100644 --- a/tests/test_health_resource.py +++ b/tests/test_health_resource.py @@ -2,22 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from contextlib import contextmanager import json -import os - - -@contextmanager -def gcs_crashstorage_envvar(): - # NOTE(willkg): we do this in a goofy way here because it means we don't actually - # have to set this in the environment which causes the app to fail at startup - # because the bucket doesn't exist - key = "CRASHMOVER_CRASHSTORAGE_CLASS" - os.environ[key] = "antenna.ext.gcs.crashstorage.GcsCrashStorage" - - yield - - del os.environ[key] class TestHealthChecks: @@ -27,20 +12,9 @@ def test_no_version(self, client, tmpdir): client.rebuild_app({"BASEDIR": str(tmpdir)}) result = client.simulate_get("/__version__") - version_info = {"cloud": "AWS"} - assert json.loads(result.content) == version_info - - def test_no_version_gcp(self, client, tmpdir): - # Set basedir here to tmpdir which we *know* doesn't have a - # version.json in it. - client.rebuild_app({"BASEDIR": str(tmpdir)}) - - with gcs_crashstorage_envvar(): - result = client.simulate_get("/__version__") - version_info = {"cloud": "GCP"} - assert json.loads(result.content) == version_info + assert json.loads(result.content) == {} - def test_version_aws(self, client, tmpdir): + def test_version(self, client, tmpdir): client.rebuild_app({"BASEDIR": str(tmpdir)}) # NOTE(willkg): The actual version.json has other things in it, @@ -50,21 +24,7 @@ def test_version_aws(self, client, tmpdir): version_path.write('{"commit": "ou812"}') result = client.simulate_get("/__version__") - version_info = {"commit": "ou812", "cloud": "AWS"} - assert json.loads(result.content) == version_info - - def test_version_gcp(self, client, tmpdir): - client.rebuild_app({"BASEDIR": str(tmpdir)}) - - # NOTE(willkg): The actual version.json has other things in it, - # but our endpoint just spits out the file verbatim, so we - # can test with whatever. - version_path = tmpdir.join("/version.json") - version_path.write('{"commit": "ou812"}') - - with gcs_crashstorage_envvar(): - result = client.simulate_get("/__version__") - version_info = {"commit": "ou812", "cloud": "GCP"} + version_info = {"commit": "ou812"} assert json.loads(result.content) == version_info def test_lb_heartbeat(self, client): diff --git a/tests/test_s3_crashstorage.py b/tests/test_s3_crashstorage.py deleted file mode 100644 index e4ff6491..00000000 --- a/tests/test_s3_crashstorage.py +++ /dev/null @@ -1,321 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import io -import logging -from unittest.mock import patch, ANY - -import botocore -import pytest - -from testlib.mini_poster import multipart_encode - - -@pytest.fixture -def mock_generate_test_filepath(): - with patch("antenna.ext.s3.connection.generate_test_filepath") as gtfp: - gtfp.return_value = "test/testwrite.txt" - yield - - -class TestS3CrashStorageIntegration: - logging_names = ["antenna"] - - def test_crash_storage(self, client, s3mock, mock_generate_test_filepath): - # .verify_write_to_bucket() writes to the bucket to verify Antenna can - # write to it and the configuration is correct - s3mock.add_step( - method="PUT", - url="http://fakes3:4569/testbucket/test/testwrite.txt", - body=b"test", - resp=s3mock.fake_response(status_code=200), - ) - - # # We want to verify these files are saved in this specific order. - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/dump_names/" - + "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b'["upload_file_minidump"]', - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url="http://fakes3:4569/testbucket/v1/dump/de1bb258-cbbf-4589-a673-34f800160918", - body=b"abcd1234", - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/raw_crash/20160918/" - + "de1bb258-cbbf-4589-a673-34f800160918" - ), - # Not going to compare the body here because it's just the raw crash - resp=s3mock.fake_response(status_code=200), - ) - data, headers = multipart_encode( - { - "uuid": "de1bb258-cbbf-4589-a673-34f800160918", - "ProductName": "Firefox", - "Version": "1.0", - "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), - } - ) - - # Rebuild the app the test client is using with relevant configuration. - client.rebuild_app( - { - "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.s3.crashstorage.S3CrashStorage", - "CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL": "http://fakes3:4569", - } - ) - - result = client.simulate_post("/submit", headers=headers, body=data) - - # Verify the collector returns a 200 status code and the crash id - # we fed it. - assert result.status_code == 200 - assert result.content == b"CrashID=bp-de1bb258-cbbf-4589-a673-34f800160918\n" - - # Assert we did the entire s3 conversation - assert s3mock.remaining_conversation() == [] - - def test_region_and_bucket_with_periods( - self, client, s3mock, mock_generate_test_filepath - ): - # .verify_write_to_bucket() writes to the bucket to verify Antenna can - # write to it and the configuration is correct - s3mock.add_step( - method="PUT", - url="http://fakes3:4569/testbucket.with.periods/test/testwrite.txt", - body=b"test", - resp=s3mock.fake_response(status_code=200), - ) - - # We want to verify these files are saved in this specific order. - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket.with.periods/v1/dump_names/" - "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b'["upload_file_minidump"]', - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket.with.periods/v1/dump/" - "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b"abcd1234", - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket.with.periods/v1/raw_crash/20160918/" - + "de1bb258-cbbf-4589-a673-34f800160918" - ), - # Not going to compare the body here because it's just the raw crash - resp=s3mock.fake_response(status_code=200), - ) - data, headers = multipart_encode( - { - "uuid": "de1bb258-cbbf-4589-a673-34f800160918", - "ProductName": "Firefox", - "Version": "1.0", - "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), - } - ) - - # Rebuild the app the test client is using with relevant configuration. - client.rebuild_app( - { - "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.s3.crashstorage.S3CrashStorage", - "CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL": "http://fakes3:4569", - "CRASHMOVER_CRASHSTORAGE_BUCKET_NAME": "testbucket.with.periods", - } - ) - - result = client.simulate_post("/submit", headers=headers, body=data) - - # Verify the collector returns a 200 status code and the crash id - # we fed it. - assert result.status_code == 200 - assert result.content == b"CrashID=bp-de1bb258-cbbf-4589-a673-34f800160918\n" - - # Assert we did the entire s3 conversation - assert s3mock.remaining_conversation() == [] - - def test_missing_bucket_halts_startup( - self, client, s3mock, mock_generate_test_filepath - ): - # .verify_write_to_bucket() writes to the bucket to verify Antenna can - # write to it and the configuration is correct - s3mock.add_step( - method="PUT", - url="http://fakes3:4569/testbucket/test/testwrite.txt", - body=b"test", - resp=s3mock.fake_response(status_code=404), - ) - - with pytest.raises(botocore.exceptions.ClientError) as excinfo: - # Rebuild the app the test client is using with relevant - # configuration. This calls .verify_write_to_bucket() which fails. - client.rebuild_app( - { - "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.s3.crashstorage.S3CrashStorage", - "CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL": "http://fakes3:4569", - } - ) - - assert ( - "An error occurred (404) when calling the PutObject operation: Not Found" - in str(excinfo.value) - ) - - # Assert we did the entire s3 conversation - assert s3mock.remaining_conversation() == [] - - def test_retrying(self, client, s3mock, caplog, mock_generate_test_filepath): - # .verify_write_to_bucket() writes to the bucket to verify Antenna can - # write to it and the configuration is correct - s3mock.add_step( - method="PUT", - url="http://fakes3:4569/testbucket/test/testwrite.txt", - body=b"test", - resp=s3mock.fake_response(status_code=200), - ) - - # Fail once with a 403, retry and then proceed. - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/dump_names/" - "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b'["upload_file_minidump"]', - resp=s3mock.fake_response(status_code=403), - ) - - # Proceed with saving files. - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/dump_names/" - "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b'["upload_file_minidump"]', - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/dump/" - "de1bb258-cbbf-4589-a673-34f800160918" - ), - body=b"abcd1234", - resp=s3mock.fake_response(status_code=200), - ) - s3mock.add_step( - method="PUT", - url=( - "http://fakes3:4569/testbucket/v1/raw_crash/" - + "20160918/de1bb258-cbbf-4589-a673-34f800160918" - ), - # Not going to compare the body here because it's just the raw crash - resp=s3mock.fake_response(status_code=200), - ) - data, headers = multipart_encode( - { - "uuid": "de1bb258-cbbf-4589-a673-34f800160918", - "ProductName": "Firefox", - "Version": "1.0", - "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), - } - ) - - # Rebuild the app the test client is using with relevant configuration. - client.rebuild_app( - { - "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.s3.crashstorage.S3CrashStorage", - "CRASHMOVER_CRASHSTORAGE_ENDPOINT_URL": "http://fakes3:4569", - } - ) - - result = client.simulate_post("/submit", headers=headers, body=data) - - # Verify the collector returns a 200 status code and the crash id - # we fed it. - assert result.status_code == 200 - assert result.content == b"CrashID=bp-de1bb258-cbbf-4589-a673-34f800160918\n" - - # Verify the retry decorator logged something - records = [ - rec for rec in caplog.record_tuples if rec[0] == "antenna.ext.s3.connection" - ] - assert records == [ - ( - "antenna.ext.s3.connection", - logging.WARNING, - ( - "S3Connection.save_file: exception An error occurred (403) " - "when calling the PutObject operation: Forbidden, retry attempt 0" - ), - ) - ] - - # Assert we did the entire s3 conversation - assert s3mock.remaining_conversation() == [] - - # FIXME(willkg): Add test for bad region - # FIXME(willkg): Add test for invalid credentials - - def test_load_crash(self, client, s3_helper): - crash_id = "de1bb258-cbbf-4589-a673-34f800160918" - data, headers = multipart_encode( - { - "uuid": crash_id, - "ProductName": "Firefox", - "Version": "1.0", - "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), - } - ) - - client.rebuild_app( - { - "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.s3.crashstorage.S3CrashStorage", - } - ) - - result = client.simulate_post("/submit", headers=headers, body=data) - assert result.status_code == 200 - - s3_crashstorage = client.get_crashmover().crashstorage - crash_report = s3_crashstorage.load_crash(crash_id) - - assert crash_report.crash_id == crash_id - assert crash_report.dumps == {"upload_file_minidump": b"abcd1234"} - assert crash_report.raw_crash == { - "uuid": crash_id, - "ProductName": "Firefox", - "Version": "1.0", - "metadata": { - "collector_notes": [], - "dump_checksums": { - "upload_file_minidump": "e9cee71ab932fde863338d08be4de9dfe39ea049bdafb342ce659ec5450b69ae" - }, - "payload": "multipart", - "payload_compressed": "0", - "payload_size": 648, - "throttle_rule": "accept_everything", - "user_agent": ANY, - }, - "submitted_timestamp": ANY, - "version": 2, - } diff --git a/tests/test_sentry.py b/tests/test_sentry.py index 996eab2c..9a367c7e 100644 --- a/tests/test_sentry.py +++ b/tests/test_sentry.py @@ -83,7 +83,6 @@ "sdk": { "integrations": [ "atexit", - "boto3", "dedupe", "excepthook", "modules", diff --git a/tests/test_sqs_crashpublish.py b/tests/test_sqs_crashpublish.py deleted file mode 100644 index b8f807db..00000000 --- a/tests/test_sqs_crashpublish.py +++ /dev/null @@ -1,134 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import io -import os - -import boto3 -from botocore.exceptions import ClientError -import pytest - -from testlib.mini_poster import multipart_encode - - -class SQSHelper: - def __init__(self): - self.client = self._build_client() - self._queues = [] - - def cleanup(self): - """Clean up SQS queues creating in tests.""" - for queue_name in self._queues: - queue_url = self.client.get_queue_url(QueueName=queue_name)["QueueUrl"] - self.client.delete_queue(QueueUrl=queue_url) - - def _build_client(self): - """Build a client.""" - session = boto3.session.Session( - aws_access_key_id=os.environ.get("CRASHMOVER_CRASHPUBLISH_ACCESS_KEY"), - aws_secret_access_key=os.environ.get( - "CRASHMOVER_CRASHPUBLISH_SECRET_ACCESS_KEY" - ), - ) - client = session.client( - service_name="sqs", - region_name=os.environ.get("CRASHMOVER_CRASHPUBLISH_REGION"), - endpoint_url=os.environ.get("CRASHMOVER_CRASHPUBLISH_ENDPOINT_URL"), - ) - return client - - def create_queue(self, queue_name): - """Create a queue.""" - self.client.create_queue(QueueName=queue_name) - self._queues.append(queue_name) - - def get_published_crashids(self, queue_name): - """Get crash ids published to the queue.""" - queue_url = self.client.get_queue_url(QueueName=queue_name)["QueueUrl"] - all_crashids = [] - while True: - resp = self.client.receive_message( - QueueUrl=queue_url, - WaitTimeSeconds=0, - VisibilityTimeout=1, - ) - msgs = resp.get("Messages", []) - if not msgs: - return all_crashids - all_crashids.extend([msg["Body"] for msg in msgs]) - - -@pytest.fixture -def sqs(): - """AWS SQS helper fixture.""" - sqs = SQSHelper() - - yield sqs - - sqs.cleanup() - - -class TestSQSCrashPublishIntegration: - def test_verify_queue_no_queue(self, client, sqs): - # Rebuild the app the test client is using with relevant configuration--this - # will call verify_queue() which will balk because the queue doesn't exist. - with pytest.raises(ClientError): - client.rebuild_app( - { - "CRASHMOVER_CRASHPUBLISH_CLASS": "antenna.ext.sqs.crashpublish.SQSCrashPublish", - "CRASHMOVER_CRASHPUBLISH_QUEUE_NAME": "test_socorro", - } - ) - - def test_verify_topic_with_queue(self, client, sqs): - queue_name = "test_socorro" - sqs.create_queue(queue_name) - - # Rebuild the app the test client is using with relevant configuration--this - # will call verify_topic() which will work fine. - client.rebuild_app( - { - "CRASHMOVER_CRASHPUBLISH_CLASS": "antenna.ext.sqs.crashpublish.SQSCrashPublish", - "CRASHMOVER_CRASHPUBLISH_QUEUE_NAME": "test_socorro", - } - ) - - # Assert "test" crash id was published - crashids = sqs.get_published_crashids(queue_name) - assert crashids == ["test"] - - def test_crash_publish(self, client, sqs): - queue_name = "test_socorro" - sqs.create_queue(queue_name) - - data, headers = multipart_encode( - { - "uuid": "de1bb258-cbbf-4589-a673-34f800160918", - "ProductName": "Firefox", - "Version": "1.0", - "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), - } - ) - - # Rebuild the app the test client is using with relevant configuration - client.rebuild_app( - { - "CRASHMOVER_CRASHPUBLISH_CLASS": "antenna.ext.sqs.crashpublish.SQSCrashPublish", - "CRASHMOVER_CRASHPUBLISH_QUEUE_NAME": "test_socorro", - } - ) - - # Slurp off the "test" crash id from verification - sqs.get_published_crashids(queue_name) - - result = client.simulate_post("/submit", headers=headers, body=data) - - # Verify the collector returns a 200 status code and the crash id - # we fed it. - assert result.status_code == 200 - assert result.content == b"CrashID=bp-de1bb258-cbbf-4589-a673-34f800160918\n" - - # Assert crash id was published - crashids = sqs.get_published_crashids(queue_name) - assert crashids == ["de1bb258-cbbf-4589-a673-34f800160918"]