diff --git a/.vscode/launch.json b/.vscode/launch.json index ddfcbacde..139913e99 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -33,6 +33,38 @@ "DIPDUP_DEBUG": "1" } }, + { + "name": "demo_squid_cloud: run", + "type": "debugpy", + "request": "launch", + "module": "dipdup", + "args": [ + "-e", + ".env", + "run" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/src/demo_squid_cloud", + "env": { + "DIPDUP_DEBUG": "1" + } + }, + { + "name": "demo_squid_cloud: init", + "type": "debugpy", + "request": "launch", + "module": "dipdup", + "args": [ + "-e", + ".env", + "init" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/src/demo_squid_cloud", + "env": { + "DIPDUP_DEBUG": "1" + } + }, { "name": "demo_evm_events: run", "type": "debugpy", diff --git a/docs/8.examples/_demos_table.md b/docs/8.examples/_demos_table.md index 5fb2893c1..dfd94c69d 100644 --- a/docs/8.examples/_demos_table.md +++ b/docs/8.examples/_demos_table.md @@ -2,6 +2,7 @@ | name | network | description | source | |-|-|-|-| | demo_blank | | Empty config for a fresh start | [link](https://github.com/dipdup-io/dipdup/tree/8.0.0b3/src/demo_blank) | +| demo_squid_cloud | | Deploying DipDup to Squid Cloud | [link](https://github.com/dipdup-io/dipdup/tree/8.0.0b3/src/demo_squid_cloud) | | demo_evm_events | EVM | ERC-20 token transfers (from event logs) | [link](https://github.com/dipdup-io/dipdup/tree/8.0.0b3/src/demo_evm_events) | | demo_evm_transactions | EVM | ERC-20 token transfers (from transactions) | [link](https://github.com/dipdup-io/dipdup/tree/8.0.0b3/src/demo_evm_transactions) | | demo_evm_uniswap | EVM | Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) | [link](https://github.com/dipdup-io/dipdup/tree/8.0.0b3/src/demo_evm_uniswap) | diff --git a/src/demo_squid_cloud/.dockerignore b/src/demo_squid_cloud/.dockerignore new file mode 100644 index 000000000..f4bdc1fff --- /dev/null +++ b/src/demo_squid_cloud/.dockerignore @@ -0,0 +1,22 @@ +# Ignore all +* + +# Add metadata and build files +!demo_squid_cloud +!pyproject.toml +!*.lock +!README.md + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/.keep \ No newline at end of file diff --git a/src/demo_squid_cloud/.gitignore b/src/demo_squid_cloud/.gitignore new file mode 100644 index 000000000..65430b412 --- /dev/null +++ b/src/demo_squid_cloud/.gitignore @@ -0,0 +1,29 @@ +# Ignore all +* +!*/ + +# Add metadata and build files +!demo_squid_cloud +!.gitignore +!.dockerignore +!py.typed +!**/Dockerfile +!**/Makefile +!**/pyproject.toml +!**/*.lock +!**/README.md +!**/.keep + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/*.env.default \ No newline at end of file diff --git a/src/demo_squid_cloud/Makefile b/src/demo_squid_cloud/Makefile new file mode 100644 index 000000000..29d16d010 --- /dev/null +++ b/src/demo_squid_cloud/Makefile @@ -0,0 +1,45 @@ +.PHONY: $(MAKECMDGOALS) +MAKEFLAGS += --no-print-directory +## +## 🚧 DipDup developer tools +## +PACKAGE=demo_squid_cloud +TAG=latest +COMPOSE=deploy/compose.yaml + +help: ## Show this help (default) + @grep -Fh "##" $(MAKEFILE_LIST) | grep -Fv grep -F | sed -e 's/\\$$//' | sed -e 's/##//' + +all: ## Run an entire CI pipeline + make format lint + +format: ## Format with all tools + make black + +lint: ## Lint with all tools + make ruff mypy + +## + +black: ## Format with black + black . + +ruff: ## Lint with ruff + ruff check --fix . + +mypy: ## Lint with mypy + mypy --no-incremental --exclude ${PACKAGE} . + +## + +image: ## Build Docker image + docker buildx build . -t ${PACKAGE}:${TAG} --load + +up: ## Start Compose stack + docker-compose -f ${COMPOSE} up -d --build + docker-compose -f ${COMPOSE} logs -f + +down: ## Stop Compose stack + docker-compose -f ${COMPOSE} down + +## \ No newline at end of file diff --git a/src/demo_squid_cloud/README.md b/src/demo_squid_cloud/README.md new file mode 100644 index 000000000..e7775e4a1 --- /dev/null +++ b/src/demo_squid_cloud/README.md @@ -0,0 +1,49 @@ +# demo_squid_cloud + +Deploying DipDup to Squid Cloud + +## Installation + +This project is based on [DipDup](https://dipdup.io), a framework for building featureful dapps. + +You need a Linux/macOS system with Python 3.12 installed. To install DipDup with pipx or use our installer: + +```shell +curl -Lsf https://dipdup.io/install.py | python3.12 +``` + +See the [Installation](https://dipdup.io/docs/installation) page for all options. + +## Usage + +Run the indexer in memory: + +```shell +dipdup run +``` + +Store data in SQLite database: + +```shell +dipdup -c . -c configs/dipdup.sqlite.yaml run +``` + +Or spawn a Compose stack with PostgreSQL and Hasura: + +```shell +cd deploy +cp .env.default .env +# Edit .env file before running +docker-compose up +``` + +## Development setup + +To set up the development environment: + +```shell +pdm install +$(pdm venv activate) +``` + +Run `make all` to run full CI check or `make help` to see other available commands. \ No newline at end of file diff --git a/src/demo_squid_cloud/__init__.py b/src/demo_squid_cloud/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/abi/.keep b/src/demo_squid_cloud/abi/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/configs/.keep b/src/demo_squid_cloud/configs/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/configs/dipdup.compose.yaml b/src/demo_squid_cloud/configs/dipdup.compose.yaml new file mode 100644 index 000000000..78fba783f --- /dev/null +++ b/src/demo_squid_cloud/configs/dipdup.compose.yaml @@ -0,0 +1,24 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: ${HASURA_ALLOW_AGGREGATIONS:-true} + select_limit: ${HASURA_SELECT_LIMIT:-10000} + camel_case: ${HASURA_CAMEL_CASE:-true} + +sentry: + dsn: ${SENTRY_DSN:-''} + environment: ${SENTRY_ENVIRONMENT:-''} + +prometheus: + host: 0.0.0.0 + +api: + host: 0.0.0.0 \ No newline at end of file diff --git a/src/demo_squid_cloud/configs/dipdup.sqlite.yaml b/src/demo_squid_cloud/configs/dipdup.sqlite.yaml new file mode 100644 index 000000000..3f99a8f15 --- /dev/null +++ b/src/demo_squid_cloud/configs/dipdup.sqlite.yaml @@ -0,0 +1,3 @@ +database: + kind: sqlite + path: ${SQLITE_PATH:-/tmp/demo_squid_cloud.sqlite} \ No newline at end of file diff --git a/src/demo_squid_cloud/configs/dipdup.squid-cloud.yaml b/src/demo_squid_cloud/configs/dipdup.squid-cloud.yaml new file mode 100644 index 000000000..b282ceac5 --- /dev/null +++ b/src/demo_squid_cloud/configs/dipdup.squid-cloud.yaml @@ -0,0 +1,25 @@ +database: + kind: postgres + host: ${DB_HOST:-db} + port: ${DB_PORT} + user: ${DB_USER:-dipdup} + password: ${DB_PASS} + database: ${DB_NAME:-dipdup} + +hasura: + url: http://${HASURA_HOST:-hasura}:8080 + admin_secret: ${HASURA_GRAPHQL_ADMIN_SECRET} + allow_aggregations: ${HASURA_ALLOW_AGGREGATIONS:-true} + select_limit: ${HASURA_SELECT_LIMIT:-10000} + camel_case: ${HASURA_CAMEL_CASE:-true} + +sentry: + dsn: ${SENTRY_DSN:-''} + environment: ${SENTRY_ENVIRONMENT:-''} + +prometheus: + host: 0.0.0.0 + port: 3000 + +api: + host: 0.0.0.0 \ No newline at end of file diff --git a/src/demo_squid_cloud/configs/dipdup.swarm.yaml b/src/demo_squid_cloud/configs/dipdup.swarm.yaml new file mode 100644 index 000000000..ac3ba068a --- /dev/null +++ b/src/demo_squid_cloud/configs/dipdup.swarm.yaml @@ -0,0 +1,24 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-demo_squid_cloud_db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-demo_squid_cloud_hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: ${HASURA_ALLOW_AGGREGATIONS:-false} + select_limit: ${HASURA_SELECT_LIMIT:-100} + camel_case: ${HASURA_CAMEL_CASE:-true} + +sentry: + dsn: ${SENTRY_DSN:-''} + environment: ${SENTRY_ENVIRONMENT:-''} + +prometheus: + host: 0.0.0.0 + +api: + host: 0.0.0.0 \ No newline at end of file diff --git a/src/demo_squid_cloud/configs/replay.yaml b/src/demo_squid_cloud/configs/replay.yaml new file mode 100644 index 000000000..c7631535e --- /dev/null +++ b/src/demo_squid_cloud/configs/replay.yaml @@ -0,0 +1,18 @@ +# To refresh existing project run `dipdup init --base --force` after modifying this file. +# To generate a new project from this replay run `dipdup new --replay `. +# +spec_version: 3.0 +replay: + dipdup_version: 8 + template: demo_squid_cloud + package: demo_squid_cloud + version: 0.0.1 + description: Deploying DipDup to Squid Cloud + license: MIT + name: John Doe + email: john_doe@example.com + postgres_image: postgres:15 + postgres_data_path: /var/lib/postgresql/data + hasura_image: hasura/graphql-engine:latest + line_length: 120 + package_manager: pdm diff --git a/src/demo_squid_cloud/deploy/.env.default b/src/demo_squid_cloud/deploy/.env.default new file mode 100644 index 000000000..368bdb537 --- /dev/null +++ b/src/demo_squid_cloud/deploy/.env.default @@ -0,0 +1,14 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_ALLOW_AGGREGATIONS=true +HASURA_CAMEL_CASE=true +HASURA_HOST=hasura +HASURA_SECRET= +HASURA_SELECT_LIMIT=10000 +POSTGRES_DB=dipdup +POSTGRES_HOST=db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN='' +SENTRY_ENVIRONMENT='' diff --git a/src/demo_squid_cloud/deploy/.keep b/src/demo_squid_cloud/deploy/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/deploy/Dockerfile b/src/demo_squid_cloud/deploy/Dockerfile new file mode 100644 index 000000000..7ebc8d45c --- /dev/null +++ b/src/demo_squid_cloud/deploy/Dockerfile @@ -0,0 +1,9 @@ +FROM dipdup/dipdup:8 +# FROM ghcr.io/dipdup-io/dipdup:8 +# FROM ghcr.io/dipdup-io/dipdup:next + +# COPY --chown=dipdup pyproject.toml README.md . +# RUN pip install . + +COPY --chown=dipdup . demo_squid_cloud +WORKDIR demo_squid_cloud \ No newline at end of file diff --git a/src/demo_squid_cloud/deploy/compose.sqlite.yaml b/src/demo_squid_cloud/deploy/compose.sqlite.yaml new file mode 100644 index 000000000..971b4aaec --- /dev/null +++ b/src/demo_squid_cloud/deploy/compose.sqlite.yaml @@ -0,0 +1,19 @@ +version: "3.8" +name: demo_squid_cloud + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.sqlite.yaml", "run"] + restart: always + env_file: .env + ports: + - 46339 + - 9000 + volumes: + - sqlite:${SQLITE_PATH:-/tmp/demo_squid_cloud.sqlite} + +volumes: + sqlite: \ No newline at end of file diff --git a/src/demo_squid_cloud/deploy/compose.swarm.yaml b/src/demo_squid_cloud/deploy/compose.swarm.yaml new file mode 100644 index 000000000..3320ff3b6 --- /dev/null +++ b/src/demo_squid_cloud/deploy/compose.swarm.yaml @@ -0,0 +1,92 @@ +version: "3.8" +name: demo_squid_cloud + +services: + dipdup: + image: ${IMAGE:-ghcr.io/dipdup-io/dipdup}:${TAG:-8} + depends_on: + - db + - hasura + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.swarm.yaml", "run"] + env_file: .env + networks: + - internal + - prometheus-private + deploy: + mode: replicated + replicas: ${INDEXER_ENABLED:-1} + labels: + - prometheus-job=${SERVICE} + - prometheus-port=8000 + placement: &placement + constraints: + - node.labels.${SERVICE} == true + logging: &logging + driver: "json-file" + options: + max-size: "10m" + max-file: "10" + tag: "\{\{.Name\}\}.\{\{.ImageID\}\}" + + db: + image: postgres:15 + volumes: + - db:/var/lib/postgresql/data + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - internal + deploy: + mode: replicated + replicas: 1 + placement: *placement + logging: *logging + + hasura: + image: hasura/graphql-engine:latest + depends_on: + - db + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@demo_squid_cloud_db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=false + - HASURA_GRAPHQL_LOG_LEVEL=warn + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + networks: + - internal + - traefik-public + deploy: + mode: replicated + replicas: 1 + labels: + - traefik.enable=true + - traefik.http.services.${SERVICE}.loadbalancer.server.port=8080 + - "traefik.http.routers.${SERVICE}.rule=Host(`${HOST}`) && (PathPrefix(`/v1/graphql`) || PathPrefix(`/api/rest`))" + - traefik.http.routers.${SERVICE}.entrypoints=http,${INGRESS:-ingress} + - "traefik.http.routers.${SERVICE}-console.rule=Host(`${SERVICE}.${SWARM_ROOT_DOMAIN}`)" + - traefik.http.routers.${SERVICE}-console.entrypoints=https + - traefik.http.middlewares.${SERVICE}-console.headers.customrequestheaders.X-Hasura-Admin-Secret=${HASURA_SECRET} + - traefik.http.routers.${SERVICE}-console.middlewares=authelia@docker,${SERVICE}-console + placement: *placement + logging: *logging + +volumes: + db: + +networks: + internal: + traefik-public: + external: true + prometheus-private: + external: true \ No newline at end of file diff --git a/src/demo_squid_cloud/deploy/compose.yaml b/src/demo_squid_cloud/deploy/compose.yaml new file mode 100644 index 000000000..3f37eff88 --- /dev/null +++ b/src/demo_squid_cloud/deploy/compose.yaml @@ -0,0 +1,55 @@ +version: "3.8" +name: demo_squid_cloud + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + restart: always + env_file: .env + ports: + - 46339 + - 9000 + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.compose.yaml", "run"] + depends_on: + - db + - hasura + + db: + image: postgres:15 + ports: + - 5432 + volumes: + - db:/var/lib/postgresql/data + restart: always + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dipdup"] + interval: 10s + timeout: 5s + retries: 5 + + hasura: + image: hasura/graphql-engine:latest + ports: + - 8080 + depends_on: + - db + restart: always + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=true + - HASURA_GRAPHQL_LOG_LEVEL=info + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + +volumes: + db: \ No newline at end of file diff --git a/src/demo_squid_cloud/deploy/sqlite.env.default b/src/demo_squid_cloud/deploy/sqlite.env.default new file mode 100644 index 000000000..1df9e7814 --- /dev/null +++ b/src/demo_squid_cloud/deploy/sqlite.env.default @@ -0,0 +1,4 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +SQLITE_PATH=/tmp/demo_squid_cloud.sqlite diff --git a/src/demo_squid_cloud/deploy/squid-cloud.env.default b/src/demo_squid_cloud/deploy/squid-cloud.env.default new file mode 100644 index 000000000..a1463c25f --- /dev/null +++ b/src/demo_squid_cloud/deploy/squid-cloud.env.default @@ -0,0 +1,15 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +DB_HOST=db +DB_NAME=dipdup +DB_PASS= +DB_PORT= +DB_USER=dipdup +HASURA_ALLOW_AGGREGATIONS=true +HASURA_CAMEL_CASE=true +HASURA_GRAPHQL_ADMIN_SECRET= +HASURA_HOST=hasura +HASURA_SELECT_LIMIT=10000 +SENTRY_DSN='' +SENTRY_ENVIRONMENT='' diff --git a/src/demo_squid_cloud/deploy/swarm.env.default b/src/demo_squid_cloud/deploy/swarm.env.default new file mode 100644 index 000000000..f8e1db62a --- /dev/null +++ b/src/demo_squid_cloud/deploy/swarm.env.default @@ -0,0 +1,14 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_ALLOW_AGGREGATIONS=false +HASURA_CAMEL_CASE=true +HASURA_HOST=demo_squid_cloud_hasura +HASURA_SECRET= +HASURA_SELECT_LIMIT=100 +POSTGRES_DB=dipdup +POSTGRES_HOST=demo_squid_cloud_db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN='' +SENTRY_ENVIRONMENT='' diff --git a/src/demo_squid_cloud/dipdup.yaml b/src/demo_squid_cloud/dipdup.yaml new file mode 100644 index 000000000..6ea1a7d56 --- /dev/null +++ b/src/demo_squid_cloud/dipdup.yaml @@ -0,0 +1,2 @@ +spec_version: 3.0 +package: demo_squid_cloud \ No newline at end of file diff --git a/src/demo_squid_cloud/graphql/.keep b/src/demo_squid_cloud/graphql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/handlers/.keep b/src/demo_squid_cloud/handlers/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/hasura/.keep b/src/demo_squid_cloud/hasura/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/hooks/.keep b/src/demo_squid_cloud/hooks/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/hooks/on_index_rollback.py b/src/demo_squid_cloud/hooks/on_index_rollback.py new file mode 100644 index 000000000..3d38655e0 --- /dev/null +++ b/src/demo_squid_cloud/hooks/on_index_rollback.py @@ -0,0 +1,16 @@ +from dipdup.context import HookContext +from dipdup.index import Index + + +async def on_index_rollback( + ctx: HookContext, + index: Index, # type: ignore[type-arg] + from_level: int, + to_level: int, +) -> None: + await ctx.execute_sql('on_index_rollback') + await ctx.rollback( + index=index.name, + from_level=from_level, + to_level=to_level, + ) \ No newline at end of file diff --git a/src/demo_squid_cloud/hooks/on_reindex.py b/src/demo_squid_cloud/hooks/on_reindex.py new file mode 100644 index 000000000..0804aae37 --- /dev/null +++ b/src/demo_squid_cloud/hooks/on_reindex.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_reindex( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_reindex') \ No newline at end of file diff --git a/src/demo_squid_cloud/hooks/on_restart.py b/src/demo_squid_cloud/hooks/on_restart.py new file mode 100644 index 000000000..2581b5be3 --- /dev/null +++ b/src/demo_squid_cloud/hooks/on_restart.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_restart( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_restart') \ No newline at end of file diff --git a/src/demo_squid_cloud/hooks/on_synchronized.py b/src/demo_squid_cloud/hooks/on_synchronized.py new file mode 100644 index 000000000..09099e4b6 --- /dev/null +++ b/src/demo_squid_cloud/hooks/on_synchronized.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_synchronized( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_synchronized') \ No newline at end of file diff --git a/src/demo_squid_cloud/models/.keep b/src/demo_squid_cloud/models/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/models/__init__.py b/src/demo_squid_cloud/models/__init__.py new file mode 100644 index 000000000..66ff2bbef --- /dev/null +++ b/src/demo_squid_cloud/models/__init__.py @@ -0,0 +1,41 @@ +import enum + +from dipdup import fields +from dipdup.models import Model + + +class ExampleModel(Model): + id = fields.IntField(primary_key=True) + array = fields.ArrayField() + big_int = fields.BigIntField() + binary = fields.BinaryField() + boolean = fields.BooleanField() + decimal = fields.DecimalField(10, 2) + date = fields.DateField() + datetime = fields.DatetimeField() + enum_ = fields.EnumField(enum.Enum) + float = fields.FloatField() + int_enum = fields.IntEnumField(enum.IntEnum) + int_ = fields.IntField() + json = fields.JSONField() + small_int = fields.SmallIntField() + text = fields.TextField() + time_delta = fields.TimeDeltaField() + time = fields.TimeField() + uuid = fields.UUIDField() + + relation: fields.ForeignKeyField['ExampleModel'] = fields.ForeignKeyField( + 'models.ExampleModel', related_name='reverse_relation' + ) + m2m_relation: fields.ManyToManyField['ExampleModel'] = fields.ManyToManyField( + 'models.ExampleModel', related_name='reverse_m2m_relation' + ) + + created_at = fields.DatetimeField(auto_now_add=True) + updated_at = fields.DatetimeField(auto_now=True) + + relation_id: int + m2m_relation_ids: list[int] + + class Meta: + abstract = True diff --git a/src/demo_squid_cloud/py.typed b/src/demo_squid_cloud/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/pyproject.toml b/src/demo_squid_cloud/pyproject.toml new file mode 100644 index 000000000..154ffc798 --- /dev/null +++ b/src/demo_squid_cloud/pyproject.toml @@ -0,0 +1,56 @@ +# generated by DipDup 8.0.0b3 +[project] +name = "demo_squid_cloud" +version = "0.0.1" +description = "Deploying DipDup to Squid Cloud" +license = { text = "MIT" } +authors = [ + { name = "John Doe", email = "john_doe@example.com" }, +] +readme = "README.md" +requires-python = ">=3.12,<3.13" +dependencies = [ + "dipdup>=8,<9", +] + +[tool.pdm.dev-dependencies] +dev = [ + "black", + "ruff", + "mypy", +] + +[tool.pdm.scripts] +help = {cmd = "make help", help = "Show this help (default)"} +all = {cmd = "make all", help = "Run an entire CI pipeline"} +format = {cmd = "make format", help = "Format with all tools"} +lint = {cmd = "make lint", help = "Lint with all tools"} +black = {cmd = "make black", help = "Format with black"} +ruff = {cmd = "make ruff", help = "Lint with ruff"} +mypy = {cmd = "make mypy", help = "Lint with mypy"} +image = {cmd = "make image", help = "Build Docker image"} +up = {cmd = "make up", help = "Start Compose stack"} +down = {cmd = "make down", help = "Stop Compose stack"} + +[tool.black] +line-length = 120 +target-version = ['py312'] +skip-string-normalization = true + +[tool.ruff] +line-length = 120 +target-version = 'py312' + +[tool.ruff.lint] +extend-select = ["B", "C4", "FA", "G", "I", "PTH", "Q", "RET", "RUF", "TCH", "UP"] +flake8-quotes = { inline-quotes = "single", multiline-quotes = "double" } +isort = { force-single-line = true} + +[tool.mypy] +python_version = "3.12" +plugins = ["pydantic.mypy"] +strict = false + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" \ No newline at end of file diff --git a/src/demo_squid_cloud/sql/.keep b/src/demo_squid_cloud/sql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/sql/on_index_rollback/.keep b/src/demo_squid_cloud/sql/on_index_rollback/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/sql/on_reindex/.keep b/src/demo_squid_cloud/sql/on_reindex/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/sql/on_restart/.keep b/src/demo_squid_cloud/sql/on_restart/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/sql/on_synchronized/.keep b/src/demo_squid_cloud/sql/on_synchronized/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_squid_cloud/squid.yaml b/src/demo_squid_cloud/squid.yaml new file mode 100644 index 000000000..95b1c442a --- /dev/null +++ b/src/demo_squid_cloud/squid.yaml @@ -0,0 +1,22 @@ +manifestVersion: subsquid.io/v0.1 +name: demo_squid_cloud +version: 1 +description: Deploying DipDup to Squid Cloud +build: +deploy: + env: + HASURA_GRAPHQL_ADMIN_SECRET: "${{ secrets.HASURA_SECRET }}" + HASURA_GRAPHQL_UNAUTHORIZED_ROLE: user + HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES: "true" + # SENTRY_DSN: "${{ secrets.SENTRY_DSN }}" + # NODE_API_KEY: "${{ secrets.NODE_API_KEY }}" + # + # Only for free tier + DIPDUP_LOW_MEMORY: "1" + addons: + postgres: + hasura: + processor: + cmd: ["dipdup", "-c", "dipdup.yaml", "-c", "configs/dipdup.squid-cloud.yaml", "run"] + init: + cmd: ["echo", "dipdup"] \ No newline at end of file diff --git a/src/demo_squid_cloud/types/.keep b/src/demo_squid_cloud/types/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 76796ada5..463e08f35 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -52,7 +52,10 @@ 'demo_tezos_token_transfers', ), 'starknet': ('demo_starknet_events',), - 'other': ('demo_blank',), + 'other': ( + 'demo_blank', + 'demo_squid_cloud', + ), } # TODO: demo_jobs @@ -367,10 +370,17 @@ def _render(answers: Answers, template_path: Path, output_path: Path, force: boo if output_path.exists() and not force: _logger.info('File `%s` already exists, skipping', output_path) + class Secrets: + """Helper for secrets in `squid.yaml`""" + + def __getattr__(self, name: str) -> str: + return '{{ secrets.' + name + ' }}' + _logger.info('Generating `%s`', output_path) template = load_template(str(template_path)) content = template.render( project={k: str(v) for k, v in answers.items()}, header=CODEGEN_HEADER, + secrets=Secrets() if template_path.name == 'squid.yaml.j2' else None, ) write(output_path, content, overwrite=force) diff --git a/src/dipdup/projects/demo_squid_cloud/configs/dipdup.squid-cloud.yaml.j2 b/src/dipdup/projects/demo_squid_cloud/configs/dipdup.squid-cloud.yaml.j2 new file mode 100644 index 000000000..b282ceac5 --- /dev/null +++ b/src/dipdup/projects/demo_squid_cloud/configs/dipdup.squid-cloud.yaml.j2 @@ -0,0 +1,25 @@ +database: + kind: postgres + host: ${DB_HOST:-db} + port: ${DB_PORT} + user: ${DB_USER:-dipdup} + password: ${DB_PASS} + database: ${DB_NAME:-dipdup} + +hasura: + url: http://${HASURA_HOST:-hasura}:8080 + admin_secret: ${HASURA_GRAPHQL_ADMIN_SECRET} + allow_aggregations: ${HASURA_ALLOW_AGGREGATIONS:-true} + select_limit: ${HASURA_SELECT_LIMIT:-10000} + camel_case: ${HASURA_CAMEL_CASE:-true} + +sentry: + dsn: ${SENTRY_DSN:-''} + environment: ${SENTRY_ENVIRONMENT:-''} + +prometheus: + host: 0.0.0.0 + port: 3000 + +api: + host: 0.0.0.0 \ No newline at end of file diff --git a/src/dipdup/projects/demo_squid_cloud/dipdup.yaml.j2 b/src/dipdup/projects/demo_squid_cloud/dipdup.yaml.j2 new file mode 100644 index 000000000..a79b49108 --- /dev/null +++ b/src/dipdup/projects/demo_squid_cloud/dipdup.yaml.j2 @@ -0,0 +1,2 @@ +spec_version: 3.0 +package: {{ project.package }} diff --git a/src/dipdup/projects/demo_squid_cloud/replay.yaml b/src/dipdup/projects/demo_squid_cloud/replay.yaml new file mode 100644 index 000000000..a737cf9b3 --- /dev/null +++ b/src/dipdup/projects/demo_squid_cloud/replay.yaml @@ -0,0 +1,5 @@ +spec_version: 3.0 +replay: + description: Deploying DipDup to Squid Cloud + package: demo_squid_cloud + template: demo_squid_cloud \ No newline at end of file diff --git a/src/dipdup/projects/demo_squid_cloud/squid.yaml.j2 b/src/dipdup/projects/demo_squid_cloud/squid.yaml.j2 new file mode 100644 index 000000000..6dad4794f --- /dev/null +++ b/src/dipdup/projects/demo_squid_cloud/squid.yaml.j2 @@ -0,0 +1,22 @@ +manifestVersion: subsquid.io/v0.1 +name: {{ project.package }} +version: 1 +description: {{ project.description }} +build: +deploy: + env: + HASURA_GRAPHQL_ADMIN_SECRET: "${{ secrets.HASURA_SECRET }}" + HASURA_GRAPHQL_UNAUTHORIZED_ROLE: user + HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES: "true" + # SENTRY_DSN: "${{ secrets.SENTRY_DSN }}" + # NODE_API_KEY: "${{ secrets.NODE_API_KEY }}" + # + # Only for free tier + DIPDUP_LOW_MEMORY: "1" + addons: + postgres: + hasura: + processor: + cmd: ["dipdup", "-c", "dipdup.yaml", "-c", "configs/dipdup.squid-cloud.yaml", "run"] + init: + cmd: ["echo", "dipdup"] \ No newline at end of file