diff --git a/Makefile b/Makefile index 6bfc0cf5..96be62a2 100644 --- a/Makefile +++ b/Makefile @@ -14,20 +14,26 @@ extract: docker compose run extractors uv run python main.py configs/arbitrum_sepolia.yaml index: - docker compose run indexer --network_name base_mainnet --protocol_name synthetix - docker compose run indexer --network_name arbitrum_mainnet --protocol_name synthetix + docker compose -f docker-compose.indexers.yml up -d indexer-arbitrum-mainnet-synthetix + docker compose -f docker-compose.indexers.yml up -d indexer-base-mainnet-synthetix + +ingest: + docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/clean_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/clean_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/import_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/import_parquet.py synths: docker compose run transformer python scripts/get_synths.py dbt: build - docker compose run transformer dbt run --target prod --profiles-dir profiles --profile synthetix + docker compose run transformer dbt run --target prod --profiles-dir profiles --profile clickhouse seed-prod: build - docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile synthetix + docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile clickhouse seed-dev: build - docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile synthetix + docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile clickhouse dbt-op: build - docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile synthetix + docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile clickhouse diff --git a/docker-compose.indexers.yml b/docker-compose.indexers.yml new file mode 100644 index 00000000..252d2d3b --- /dev/null +++ b/docker-compose.indexers.yml @@ -0,0 +1,47 @@ +services: + indexer-arbitrum-mainnet-synthetix: + build: + context: ./indexers + dockerfile: Dockerfile + restart: always + networks: + - data + env_file: + - .env + environment: + - NETWORK_NAME=arbitrum_mainnet + - PROTOCOL_NAME=synthetix + volumes: + - ./parquet-data:/parquet-data + + indexer-base-mainnet-synthetix: + build: + context: ./indexers + dockerfile: Dockerfile + restart: always + networks: + - data + env_file: + - .env + environment: + - NETWORK_NAME=base_mainnet + - PROTOCOL_NAME=synthetix + volumes: + - ./parquet-data:/parquet-data + + indexer-listener: + build: + context: ./indexers + dockerfile: Dockerfile.listener + networks: + - data + restart: always + environment: + PYTHONUNBUFFERED: 1 + volumes: + - ./parquet-data:/parquet-data + +networks: + data: + external: + name: data_data diff --git a/docker-compose.yml b/docker-compose.yml index 039749e4..321e4c61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,33 +42,6 @@ services: cpus: "4.0" memory: 8192M - indexer: - build: - context: ./indexers - dockerfile: Dockerfile - networks: - - data - restart: always - env_file: - - .env - environment: - NETWORK_NAME: base_mainnet - PROTOCOL_NAME: synthetix - volumes: - - ./parquet-data:/parquet-data - - indexer-listener: - build: - context: ./indexers - dockerfile: Dockerfile.listener - networks: - - data - restart: always - environment: - PYTHONUNBUFFERED: 1 - volumes: - - ./parquet-data:/parquet-data - transformer: image: ghcr.io/synthetixio/data/transformer:${VERSION} build: