diff --git a/.env b/.env index 17036f4d54..6c5f2c3981 100644 --- a/.env +++ b/.env @@ -9,6 +9,7 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +VROOM_IMAGE=getsentry/vroom:nightly WAL2JSON_VERSION=latest HEALTHCHECK_INTERVAL=30s HEALTHCHECK_TIMEOUT=60s diff --git a/_integration-test/fixtures/envelope-with-profile b/_integration-test/fixtures/envelope-with-profile new file mode 100644 index 0000000000..299ef6a538 --- /dev/null +++ b/_integration-test/fixtures/envelope-with-profile @@ -0,0 +1,5 @@ +{"event_id":"66578634d48d433db0ad52882d1efe5b","sent_at":"2023-05-17T14:54:31.057Z","sdk":{"name":"sentry.javascript.node","version":"7.46.0"},"trace":{"environment":"production","transaction":"fib: sourcemaps here","public_key":"05ab86aebbe14a24bcab62caa839cf27","trace_id":"33321bfbd5304bcc9663d1b53b08f84e","sample_rate":"1"}} +{"type":"transaction"} +{"contexts":{"profile":{"profile_id":"e73aaf1f29b24812be60132f32d09f92"},"trace":{"op":"test","span_id":"b38f2b24537c3858","trace_id":"33321bfbd5304bcc9663d1b53b08f84e"},"runtime":{"name":"node","version":"v16.16.0"},"app":{"app_start_time":"2023-05-17T14:54:27.678Z","app_memory":57966592},"os":{"kernel_version":"22.3.0","name":"macOS","version":"13.2","build":"22D49"},"device":{"boot_time":"2023-05-12T15:08:41.047Z","arch":"arm64","memory_size":34359738368,"free_memory":6861651968,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-US","timezone":"America/New_York"}},"spans":[],"start_timestamp":1684335267.744,"tags":{},"timestamp":1684335271.033,"transaction":"fib: sourcemaps here","type":"transaction","transaction_info":{"source":"custom"},"platform":"node","server_name":"TK6G745PW1.local","event_id":"66578634d48d433db0ad52882d1efe5b","environment":"production","sdk":{"integrations":["InboundFilters","FunctionToString","Console","Http","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","Modules","RequestData","LinkedErrors","ProfilingIntegration"],"name":"sentry.javascript.node","version":"7.46.0","packages":[{"name":"npm:@sentry/node","version":"7.46.0"}]},"debug_meta":{"images":[]},"modules":{}} +{"type":"profile"} +{"event_id":"e73aaf1f29b24812be60132f32d09f92","timestamp":"2023-05-17T14:54:27.744Z","platform":"node","version":"1","release":"","environment":"production","runtime":{"name":"node","version":"16.16.0"},"os":{"name":"darwin","version":"22.3.0","build_number":"Darwin Kernel Version 22.3.0: Thu Jan 5 20:48:54 PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6000"},"device":{"locale":"en_US.UTF-8","model":"arm64","manufacturer":"Darwin","architecture":"arm64","is_emulator":false},"debug_meta":{"images":[]},"profile":{"samples":[{"stack_id":0,"thread_id":"0","elapsed_since_start_ns":125000},{"stack_id":0,"thread_id":"0","elapsed_since_start_ns":13958000}],"frames":[{"lineno":14129,"colno":17,"function":"startProfiling","abs_path":"/Users/jonasbadalic/code/node-profiler/lib/index.js"}],"stacks":[[0]],"thread_metadata":{"0":{"name":"main"}}},"transaction":{"name":"fib: sourcemaps here","id":"66578634d48d433db0ad52882d1efe5b","trace_id":"33321bfbd5304bcc9663d1b53b08f84e","active_thread_id":"0"}} diff --git a/_integration-test/run.sh b/_integration-test/run.sh index 338ba5c72f..b015de7ca5 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -135,6 +135,18 @@ $dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py source _integration-test/custom-ca-roots/teardown.sh echo "${_endgroup}" +echo "${_group}Test that profiling work ..." +echo "Sending a test profile..." +PROFILE_FIXTURE_PATH="$(git rev-parse --show-toplevel)/_integration-test/fixtures/envelope-with-profile" +curl -sf --data-binary @$PROFILE_FIXTURE_PATH -H 'Content-Type: application/x-sentry-envelope' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/envelope/" -o /dev/null + +printf "Getting the test profile back" +PROFILE_ID="$(jq -r -n --slurpfile profile $PROFILE_FIXTURE_PATH '$profile[4].event_id')" +PROFILE_PATH="api/0/projects/sentry/sentry/profiling/raw_profiles/$PROFILE_ID/" +timeout 60 bash -c 'until $(sentry_api_request "$PROFILE_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' +echo " got it!" +echo "${_endgroup}" + # Table formatting based on https://stackoverflow.com/a/39144364 COMPOSE_PS_OUTPUT=$(docker compose ps --format json | jq -r \ '.[] | diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh index 88931964e3..6b1176c5c7 100755 --- a/_unit-test/create-docker-volumes-test.sh +++ b/_unit-test/create-docker-volumes-test.sh @@ -21,8 +21,6 @@ before=$(get_volumes) test "$before" == "" || test "$before" == "$expected_volumes" -source install/create-docker-volumes.sh -source install/create-docker-volumes.sh source install/create-docker-volumes.sh after=$(get_volumes) diff --git a/docker-compose.yml b/docker-compose.yml index 60988a0637..60e7cab92f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,12 +52,15 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default symbolicator: <<: *depends_on-default + vroom: + <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" + VROOM: "http://vroom:8085" # Force everything to use the system CA bundle # This is mostly needed to support installing custom CA certs # This one is used by botocore @@ -283,6 +286,12 @@ services: snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900 + snuba-profiling-profiles-consumer: + <<: *snuba_defaults + command: consumer --storage profiles --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + snuba-profiling-functions-consumer: + <<: *snuba_defaults + command: consumer --storage functions_raw --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" @@ -329,10 +338,12 @@ services: ingest-replay-recordings: <<: *sentry_defaults command: run ingest-replay-recordings + ingest-profiles: + <<: *sentry_defaults + command: run ingest-profiles --no-strict-offset-reset post-process-forwarder-errors: <<: *sentry_defaults command: run post-process-forwarder --entity errors - post-process-forwarder-transactions: <<: *sentry_defaults command: run post-process-forwarder --entity transactions --commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group @@ -384,6 +395,19 @@ services: <<: *depends_on-healthy web: <<: *depends_on-healthy + vroom: + <<: *restart_policy + image: "$VROOM_IMAGE" + environment: + SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092" + SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092" + SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles + SENTRY_SNUBA_HOST: "http://snuba-api:1218" + volumes: + - sentry-vroom:/var/lib/sentry-profiles + depends_on: + kafka: + <<: *depends_on-healthy volumes: # These store application data that should persist across restarts. sentry-data: @@ -400,8 +424,12 @@ volumes: external: true sentry-symbolicator: external: true - + # This volume stores profiles and should be persisted. + # Not being external will still persist data across restarts. + # It won't persist if someone does a docker compose down -v. + sentry-vroom: # These store ephemeral data that needn't persist across restarts. + # That said, volumes will be persisted across restarts until they are deleted. sentry-secrets: sentry-smtp: sentry-nginx-cache: diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh index 78534ac3d0..79548a42ba 100644 --- a/install/create-kafka-topics.sh +++ b/install/create-kafka-topics.sh @@ -3,7 +3,7 @@ echo "${_group}Creating additional Kafka topics ..." # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) -NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings" +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 61ea042ad2..bdd166bf92 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -7,7 +7,7 @@ NEW_VERSION="$2" WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env -sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env +sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index f399e6fbd4..c04753fc37 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -269,6 +269,7 @@ def get_internal_network(): "organizations:performance-view", "organizations:advanced-search", "organizations:session-replay", + "organizations:profiling", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups",