Create indexers-v2 docker baseline

docker-compose.yml

@@ -26,6 +26,24 @@ services:
           cpus: "4.0"
           memory: 8192M
 
+  indexers-v2:
+    build:
+      context: ./indexers-v2
+      dockerfile: Dockerfile
+    networks:
+      - data
+    depends_on:
+      - db
+    restart: always
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: base_mainnet
+      DB_PASS: $PG_PASSWORD
+      GQL_PORT: 4350
+      RPC_ENDPOINT: $RPC_ENDPOINT
+      NETWORK_NAME: $NETWORK_NAME 
+
   eth-mainnet-processor:
     image: ghcr.io/synthetixio/data/indexer:${VERSION}-mainnet
     build:

indexers-v2/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:16-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+COPY requirements.txt .
+
+RUN apk add --no-cache python3 py3-pip
+RUN apk add --no-cache build-base && npm ci && apk del build-base
+
+COPY . .
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+ENTRYPOINT ["sh", "-c", "python3 main.py --network_name \"$NETWORK_NAME\" --rpc_endpoint \"$RPC_ENDPOINT\" && npm run generate:processor && npm run build && npm run generate:migration && npm run start"]

indexers-v2/commands.json

@@ -0,0 +1,93 @@
+{
+    "$schema": "https://cdn.subsquid.io/schemas/commands.json",
+    "commands": {
+      "clean": {
+        "description": "delete all build artifacts",
+        "cmd": ["npx", "--yes", "rimraf", "lib"]
+      },
+      "generate": {
+        "description": "Generate a squid from an ABI file",
+        "cmd": ["squid-gen-abi"]
+      },
+      "squid-gen-abi": {
+        "description": "Generate a squid from an ABI file",
+        "cmd": ["squid-gen-abi"],
+        "hidden": true
+      },
+      "build": {
+        "description": "Build the squid project",
+        "deps": ["clean"],
+        "cmd": ["tsc"]
+      },
+      "up": {
+        "description": "Start a PG database",
+        "cmd": ["docker-compose", "up", "-d"]
+      },
+      "down": {
+        "description": "Drop a PG database",
+        "cmd": ["docker-compose", "down"]
+      },
+      "migration:apply": {
+        "description": "Apply the DB migrations",
+        "cmd": ["squid-typeorm-migration", "apply"]
+      },
+      "migration:generate": {
+        "description": "Generate a DB migration matching the TypeORM entities",
+        "deps": ["build", "migration:clean"],
+        "cmd": ["squid-typeorm-migration", "generate"],
+      },
+      "migration:clean": {
+        "description": "Clean the migrations folder",
+        "cmd": ["npx", "--yes", "rimraf", "./db/migrations"],
+      },
+      "migration": {
+        "deps": ["build"],
+        "cmd": ["squid-typeorm-migration", "generate"],
+        "hidden": true
+      },
+      "codegen": {
+        "description": "Generate TypeORM entities from the schema file",
+        "cmd": ["squid-typeorm-codegen"]
+      },
+      "typegen": {
+        "description": "Generate data access classes for an ABI file(s) in the ./abi folder",
+        "cmd": ["squid-evm-typegen", "./src/abi", {"glob": "./abi/*.json"}, "--multicall"]
+      },
+      "process": {
+        "description": "Load .env and start the squid processor",
+        "deps": ["build", "migration:apply"],
+        "cmd": ["node", "--require=dotenv/config", "lib/main.js"]
+      },
+      "process:prod": {
+        "description": "Start the squid processor",
+        "deps": ["migration:apply"],
+        "cmd": ["node", "lib/main.js"],
+        "hidden": true
+      },
+      "serve": {
+        "description": "Start the GraphQL API server",
+        "cmd": ["squid-graphql-server"]
+      },
+      "serve:prod": {
+        "description": "Start the GraphQL API server with caching and limits",
+        "cmd": ["squid-graphql-server",
+          "--dumb-cache", "in-memory",
+          "--dumb-cache-ttl", "1000",
+          "--dumb-cache-size", "100",
+          "--dumb-cache-max-age", "1000" ]
+      },
+      "check-updates": {
+        "cmd": ["npx", "--yes", "npm-check-updates", "--filter=/subsquid/", "--upgrade"],
+        "hidden": true
+      },
+      "bump": {
+        "description": "Bump @subsquid packages to the latest versions",
+        "deps": ["check-updates"],
+        "cmd": ["npm", "i", "-f"]
+      },
+      "open": {
+        "description": "Open a local browser window",
+        "cmd": ["npx", "--yes", "opener"]
+      }
+    }
+  }

indexers-v2/main.py

@@ -0,0 +1,122 @@
+import json
+import sys
+import os
+import argparse
+import yaml
+from synthetix import Synthetix
+
+
+def get_contract_data(snx, contract_name, package):
+    contract = snx.contracts[package][contract_name]
+    return {
+        "name": contract_name,
+        "address": contract["address"],
+        "abi": contract["abi"],
+    }
+
+
+def save_abi(abi, filename):
+    os.makedirs("abi", exist_ok=True)
+    with open(f"abi/{filename}", "w") as file:
+        json.dump(abi, file, indent=2)
+
+
+def create_squidgen_config(rpc_url, archive_url, contracts_info):
+    config = {
+        "archive": archive_url,
+        "finalityConfirmation": 1,
+        "chain": {"url": rpc_url, "rateLimit": 10},
+        "target": {"type": "postgres"},
+        "contracts": [],
+    }
+
+    for contract in contracts_info:
+        contract_config = {
+            "name": contract["name"],
+            "address": contract["address"],
+            "abi": f"./abi/{contract['name']}.json",
+            "events": True,
+            "functions": False,
+        }
+        config["contracts"].append(contract_config)
+        save_abi(contract["abi"], f"{contract['name']}.json")
+
+    return config
+
+
+def create_squid_config(network_name):
+    return {
+        "manifestVersion": "subsquid.io/v0.1",
+        "name": network_name,
+        "version": 1,
+        "description": "A squid indexer generated from an ABI template",
+        "build": None,
+        "deploy": {
+            "addons": {"postgres": None},
+            "processor": {"cmd": ["node", "lib/main"]},
+            "api": {
+                "cmd": [
+                    "npx",
+                    "squid-graphql-server",
+                    "--dumb-cache",
+                    "in-memory",
+                    "--dumb-cache-ttl",
+                    "1000",
+                    "--dumb-cache-size",
+                    "100",
+                    "--dumb-cache-max-age",
+                    "1000",
+                ]
+            },
+        },
+    }
+
+
+def write_yaml(config, filename):
+    with open(filename, "w") as file:
+        yaml.dump(config, file, default_flow_style=False)
+
+
+def load_network_config(network_name):
+    with open("networks.yaml", "r") as file:
+        networks = yaml.safe_load(file)
+    return networks.get(network_name)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate Squid configuration files for a given network"
+    )
+    parser.add_argument("--network_name", type=str, help="Network name", required=True)
+    parser.add_argument("--rpc_endpoint", type=str, help="RPC URL", required=True)
+    args = parser.parse_args()
+
+    network_config = load_network_config(args.network_name)
+
+    if not network_config:
+        print(f"Network '{args.network_name}' not found in networks.yaml")
+        sys.exit(1)
+
+    rpc_endpoint = os.environ.get("RPC_ENDPOINT")
+    if not rpc_endpoint:
+        print("RPC_ENDPOINT environment variable is not set")
+        sys.exit(1)
+    archive_url = network_config["archive_url"]
+    contracts_to_include = network_config["contracts"]
+
+    snx = Synthetix(rpc_endpoint)
+
+    contracts_info = [
+        get_contract_data(snx, contract_name, package)
+        for contract_name, package in contracts_to_include
+    ]
+
+    squidgen_config = create_squidgen_config(rpc_endpoint, archive_url, contracts_info)
+    write_yaml(squidgen_config, "squidgen.yaml")
+
+    squid_config = create_squid_config(args.network_name)
+    write_yaml(squid_config, "squid.yaml")
+
+    print(
+        f"squidgen.yaml, squid.yaml, and ABI files have been generated for {args.network_name}"
+    )

indexers-v2/networks.yaml

@@ -0,0 +1,17 @@
+base_mainnet:
+  archive_url: "https://v2.archive.subsquid.io/network/base-mainnet"
+  contracts:
+    - ["CoreProxy", "system"]
+    - ["AccountProxy", "system"]
+    - ["SpotMarketProxy", "spotFactory"]
+    - ["PerpsMarketProxy", "perpsFactory"]
+    - ["PerpsAccountProxy", "perpsFactory"]
+
+arbitrum_mainnet:
+  archive_url: "https://v2.archive.subsquid.io/network/arbitrum-one"
+  contracts:
+    - ["CoreProxy", "system"]
+    - ["AccountProxy", "system"]
+    - ["SpotMarketProxy", "spotFactory"]
+    - ["PerpsMarketProxy", "perpsFactory"]
+    - ["PerpsAccountProxy", "perpsFactory"]