From f5ee08eccf77a4fd5d2fc8aad8f5203947ed7f00 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:26:16 +0000 Subject: [PATCH 1/5] Bumping sdk and jit dependencies to 2.100.0-beta.11 and 0.11.36 --- package.json | 4 ++-- yarn.lock | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index bde761ea..8e32f7e9 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/jit-proxy": "0.11.35", - "@drift-labs/sdk": "2.100.0-beta.10", + "@drift-labs/jit-proxy": "0.11.36", + "@drift-labs/sdk": "2.100.0-beta.11", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 6e3a5758..5037e076 100644 --- a/yarn.lock +++ b/yarn.lock @@ -197,19 +197,19 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/jit-proxy@0.11.35": - version "0.11.35" - resolved "https://registry.yarnpkg.com/@drift-labs/jit-proxy/-/jit-proxy-0.11.35.tgz#24d900ca3d9a1c5aef7763b57f1c1034eb763cd4" - integrity sha512-+TcA2J3Pkm6u/t31JrUGdHsEgHYTfSfhrfeleMVdmc1WzQpAm6ZyaSL/e9IQTRF/mwcwZZvHA603pcxKUdzLRw== +"@drift-labs/jit-proxy@0.11.36": + version "0.11.36" + resolved "https://registry.yarnpkg.com/@drift-labs/jit-proxy/-/jit-proxy-0.11.36.tgz#6a6a505b08d68e33ef614156b38f6273f46c2157" + integrity sha512-rnZ3wCnLILKArgRADYPSxK1mKyjoyGO3iTWG+VWhbNg2jtYuiDDun0U3BiCel8khsbif6Y1MJIkcHhkAmbOFww== dependencies: "@coral-xyz/anchor" "^0.26.0" - "@drift-labs/sdk" "2.100.0-beta.10" + "@drift-labs/sdk" "2.100.0-beta.11" "@solana/web3.js" "1.91.7" -"@drift-labs/sdk@2.100.0-beta.10": - version "2.100.0-beta.10" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.100.0-beta.10.tgz#b7d867134af00ef1adde5512ff98342623438539" - integrity sha512-sCYcOCP4IXXK+i6+MdbORYTSq4cq1aFvwF4q6QjsKEDXvRXuP/6AJU/nU0Ijwzw7+CW2HlPZbXQTDwiQuwHYWQ== +"@drift-labs/sdk@2.100.0-beta.11": + version "2.100.0-beta.11" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.100.0-beta.11.tgz#06dfa9f2303539c8fcce7868d09ad2b9b518a55c" + integrity sha512-UTPAfVGnob/RWSg/siOkSe4eZLWFIEn0jCME1olgcXRX2u3bzpmerIUXa6N728pdrWxvkF8r/FOlLOnujVgakQ== dependencies: "@coral-xyz/anchor" "0.28.0" "@coral-xyz/anchor-30" "npm:@coral-xyz/anchor@0.30.1" From f9f149b11915144f3000a6fb0c0423a0f7963655 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 12 Nov 2024 09:39:19 -0800 Subject: [PATCH 2/5] add paritionied markets for bots --- src/bots/pythCranker.ts | 17 +++++- src/config.ts | 47 +++------------ src/index.ts | 130 ++++------------------------------------ src/utils.ts | 98 ++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 161 deletions(-) diff --git a/src/bots/pythCranker.ts b/src/bots/pythCranker.ts index 2125224d..2f04e17b 100644 --- a/src/bots/pythCranker.ts +++ b/src/bots/pythCranker.ts @@ -128,7 +128,20 @@ export class PythCrankerBot implements Bot { await this.driftClient.fetchMarketLookupTableAccount() ); - for (const marketConfig of PerpMarkets[this.globalConfig.driftEnv]) { + const perpMarketIndexes = this.driftClient + .getPerpMarketAccounts() + .map((market) => market.marketIndex); + const perpMarketConfigs = PerpMarkets[this.globalConfig.driftEnv].filter( + (market) => perpMarketIndexes.includes(market.marketIndex) + ); + const spotMarketIndexes = this.driftClient + .getSpotMarketAccounts() + .map((market) => market.marketIndex); + const spotMarketConfigs = SpotMarkets[this.globalConfig.driftEnv].filter( + (market) => spotMarketIndexes.includes(market.marketIndex) + ); + + for (const marketConfig of perpMarketConfigs) { const feedId = marketConfig.pythFeedId; if (!feedId) { logger.warn(`No pyth feed id for market ${marketConfig.symbol}`); @@ -165,7 +178,7 @@ export class PythCrankerBot implements Bot { }); } - for (const marketConfig of SpotMarkets[this.globalConfig.driftEnv]) { + for (const marketConfig of spotMarketConfigs) { if ( this.feedIdsToCrank.findIndex( (feedId) => feedId.baseSymbol === marketConfig.symbol diff --git a/src/config.ts b/src/config.ts index 37bca848..91710897 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import YAML from 'yaml'; import { loadCommaDelimitToArray, loadCommaDelimitToStringArray, + parsePositiveIntArray, } from './utils'; import { OrderExecutionAlgoType } from './types'; import { @@ -21,17 +22,6 @@ export type BaseBotConfig = { runOnce?: boolean; }; -export type JitMakerConfig = BaseBotConfig & { - subaccounts?: Array; - marketType: string; - /// @deprecated, use {@link JitMakerConfig.marketIndexes} and {@link JitMakerConfig.marketType} - perpMarketIndicies?: Array; - marketIndexes?: Array; - targetLeverage?: number; - aggressivenessBps?: number; - jitCULimit?: number; -}; - export type UserPnlSettlerConfig = BaseBotConfig & { /// perp market indexes to filter for settling pnl perpMarketIndicies?: Array; @@ -137,14 +127,12 @@ export type BotConfigMap = { trigger?: BaseBotConfig; liquidator?: LiquidatorConfig; floatingMaker?: BaseBotConfig; - jitMaker?: JitMakerConfig; ifRevenueSettler?: BaseBotConfig; fundingRateUpdater?: BaseBotConfig; userPnlSettler?: UserPnlSettlerConfig; userLpSettler?: BaseBotConfig; userIdleFlipper?: BaseBotConfig; markTwapCrank?: BaseBotConfig; - uncrossArb?: BaseBotConfig; pythCranker?: PythCrankerBotConfig; switchboardCranker?: SwitchboardCrankerBotConfig; swiftTaker?: BaseBotConfig; @@ -160,6 +148,10 @@ export interface GlobalConfig { hermesEndpoint?: string; numNonActiveOraclesToPush?: number; + // Optional to specify markets loaded by drift client + perpMarketsToLoad?: Array; + spotMarketsToLoad?: Array; + /// helius endpoint to use helius priority fee strategy heliusEndpoint?: string; /// additional rpc endpoints to send transactions to @@ -232,6 +224,9 @@ const defaultConfig: Partial = { debug: false, subaccounts: [0], + perpMarketsToLoad: parsePositiveIntArray(process.env.PERP_MARKETS_TO_LOAD), + spotMarketsToLoad: parsePositiveIntArray(process.env.SPOT_MARKETS_TO_LOAD), + eventSubscriberPollingInterval: 5000, bulkAccountLoaderPollingInterval: 5000, @@ -455,21 +450,6 @@ export function loadConfigFromOpts(opts: any): Config { runOnce: opts.runOnce ?? false, }; } - if (opts.jitMaker) { - config.enabledBots.push('jitMaker'); - config.botConfigs!.jitMaker = { - dryRun: opts.dryRun ?? false, - botId: process.env.BOT_ID ?? 'jitMaker', - metricsPort: 9464, - runOnce: opts.runOnce ?? false, - perpMarketIndicies: loadCommaDelimitToArray(opts.perpMarketIndicies) ?? [ - 0, - ], - subaccounts: loadCommaDelimitToArray(opts.subaccounts) ?? [0], - marketType: opts.marketType ?? 'perp', - targetLeverage: 1, - }; - } if (opts.ifRevenueSettler) { config.enabledBots.push('ifRevenueSettler'); config.botConfigs!.ifRevenueSettler = { @@ -528,17 +508,6 @@ export function loadConfigFromOpts(opts: any): Config { runOnce: opts.runOnce ?? false, }; } - - if (opts.uncrossArb) { - config.enabledBots.push('uncrossArb'); - config.botConfigs!.uncrossArb = { - dryRun: opts.dryRun ?? false, - botId: process.env.BOT_ID ?? 'uncrossArb', - metricsPort: 9464, - runOnce: opts.runOnce ?? false, - }; - } - return mergeDefaults(defaultConfig, config) as Config; } diff --git a/src/index.ts b/src/index.ts index a9992e3b..a6a42d39 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,10 +24,7 @@ import { TokenFaucet, DriftClientSubscriptionConfig, LogProviderConfig, - getMarketsAndOraclesForSubscription, - AuctionSubscriber, FastSingleTxSender, - OracleInfo, UserMap, Wallet, RetryTxSender, @@ -48,7 +45,6 @@ import { constants } from './types'; import { FillerBot } from './bots/filler'; import { SpotFillerBot } from './bots/spotFiller'; import { TriggerBot } from './bots/trigger'; -import { JitMaker } from './bots/jitMaker'; import { LiquidatorBot } from './bots/liquidator'; import { FloatingPerpMakerBot } from './bots/floatingMaker'; import { Bot } from './types'; @@ -62,6 +58,7 @@ import { TOKEN_FAUCET_PROGRAM_ID, getWallet, loadKeypair, + getMarketsAndOracleInfosToLoad, } from './utils'; import { Config, @@ -71,9 +68,7 @@ import { } from './config'; import { FundingRateUpdaterBot } from './bots/fundingRateUpdater'; import { FillerLiteBot } from './bots/fillerLite'; -import { JitProxyClient, JitterShotgun } from '@drift-labs/jit-proxy/lib'; import { MakerBidAskTwapCrank } from './bots/makerBidAskTwapCrank'; -import { UncrossArbBot } from './bots/uncrossArbBot'; import { BundleSender } from './bundleSender'; import { DriftStateWatcher, StateChecks } from './driftStateWatcher'; import { webhookMessage } from './webhook'; @@ -382,16 +377,15 @@ const runBot = async () => { // keeping these arrays undefined will prompt DriftClient to call `findAllMarketAndOracles` // and load all markets and oracle accounts from on-chain - let perpMarketIndexes: number[] | undefined; - let spotMarketIndexes: number[] | undefined; - let oracleInfos: OracleInfo[] | undefined; - if (configHasBot(config, 'jitMaker')) { - ({ perpMarketIndexes, spotMarketIndexes, oracleInfos } = - getMarketsAndOraclesForSubscription(config.global.driftEnv!)); - } const marketLookupTable = config.global?.lutPubkey ? new PublicKey(config.global.lutPubkey) : new PublicKey(configs[config.global.driftEnv!].MARKET_LOOKUP_TABLE); + const marketsAndOracleInfos = getMarketsAndOracleInfosToLoad( + sdkConfig, + config.global.perpMarketsToLoad, + config.global.spotMarketsToLoad + ); + const oracleInfos = marketsAndOracleInfos.oracleInfos; const driftClientConfig = { connection, wallet, @@ -400,9 +394,9 @@ const runBot = async () => { accountSubscription, env: config.global.driftEnv, userStats: true, - perpMarketIndexes, - spotMarketIndexes, - oracleInfos, + perpMarketIndexes: marketsAndOracleInfos.perpMarketIndicies, + spotMarketIndexes: marketsAndOracleInfos.spotMarketIndicies, + oracleInfos: oracleInfos.length > 0 ? oracleInfos : undefined, activeSubAccountId: config.global.subaccounts![0], subAccountIds: config.global.subaccounts ?? [0], txVersion: 0 as TransactionVersion, @@ -696,60 +690,6 @@ const runBot = async () => { ); } - let auctionSubscriber: AuctionSubscriber | undefined = undefined; - let jitter: JitterShotgun | undefined = undefined; - if (configHasBot(config, 'jitMaker')) { - // Subscribe to drift client - - needCheckDriftUser = true; - needForceCollateral = true; - needPriorityFeeSubscriber = true; - - const jitProxyClient = new JitProxyClient({ - driftClient, - programId: new PublicKey(sdkConfig.JIT_PROXY_PROGRAM_ID!), - }); - - auctionSubscriber = new AuctionSubscriber({ - driftClient, - opts: { commitment: stateCommitment }, - }); - await auctionSubscriber.subscribe(); - - jitter = new JitterShotgun({ - auctionSubscriber, - driftClient, - jitProxyClient, - }); - await jitter.subscribe(); - - const txSenderConnection = new Connection(endpoint, { - wsEndpoint: wsEndpoint, - commitment: stateCommitment, - disableRetryOnRateLimit: true, - }); - driftClient.txSender = new FastSingleTxSender({ - connection: txSenderConnection, - wallet, - blockhashRefreshInterval: 1000, - opts: { - preflightCommitment: 'processed', - skipPreflight: false, - commitment: 'processed', - }, - }); - - bots.push( - new JitMaker( - driftClient, - jitter, - config.botConfigs!.jitMaker!, - config.global.driftEnv!, - priorityFeeSubscriber - ) - ); - } - if (configHasBot(config, 'liquidator')) { needCheckDriftUser = true; needUserMapSubscribe = true; @@ -872,33 +812,11 @@ const runBot = async () => { ); } - if (configHasBot(config, 'uncrossArb')) { - needCheckDriftUser = true; - needPriorityFeeSubscriber = true; - const jitProxyClient = new JitProxyClient({ - driftClient, - programId: new PublicKey(sdkConfig.JIT_PROXY_PROGRAM_ID!), - }); - - bots.push( - new UncrossArbBot( - driftClient, - jitProxyClient, - slotSubscriber, - config.botConfigs!.uncrossArb!, - config.global.driftEnv!, - priorityFeeSubscriber - ) - ); - } - // Run subscribe functions once if ( needCheckDriftUser || needForceCollateral || eventSubscriber || - auctionSubscriber || - jitter || needUserMapSubscribe || needPythPriceSubscriber || needDriftStateWatcher @@ -932,34 +850,6 @@ const runBot = async () => { logger.info(`userMap.subscribe took: ${hrEnd[0]}s ${hrEnd[1] / 1e6}ms`); } - logger.info( - `Checking if need auctionSubscriber: ${auctionSubscriber !== undefined}` - ); - if (auctionSubscriber) { - const hrStart = process.hrtime(); - await auctionSubscriber.subscribe(); - const hrEnd = process.hrtime(hrStart); - logger.info( - `auctionSubscriber.subscribe took: ${hrEnd[0]}s ${hrEnd[1] / 1e6}ms` - ); - } - - logger.info(`Checking if need jitter: ${jitter !== undefined}`); - if (jitter) { - const freeCollateral = driftClient - .getUser() - .getFreeCollateral('Maintenance'); - if (freeCollateral.isZero()) { - throw new Error( - `No collateral in account, collateral is required to run JitMakerBot, run with --force-deposit flag to deposit collateral` - ); - } - const hrStart = process.hrtime(); - await jitter.subscribe(); - const hrEnd = process.hrtime(hrStart); - logger.info(`jitter.subscribe took: ${hrEnd[0]}s ${hrEnd[1] / 1e6}ms`); - } - logger.info(`Checking if need pythConnection: ${needPythPriceSubscriber}`); if (needPythPriceSubscriber && pythPriceSubscriber) { const feedIds: string[] = Array.from( diff --git a/src/utils.ts b/src/utils.ts index ebec05b6..ff2ef031 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -42,6 +42,7 @@ import { DRIFT_ORACLE_RECEIVER_ID, OpenbookV2FulfillmentConfigAccount, OpenbookV2Subscriber, + OracleInfo, } from '@drift-labs/sdk'; import { NATIVE_MINT, @@ -131,6 +132,20 @@ export function loadCommaDelimitToArray(str: string): number[] { } } +export function parsePositiveIntArray( + intArray: string | undefined, + separator = ',' +): number[] | undefined { + if (!intArray) { + return undefined; + } + return intArray + .split(separator) + .map((s) => s.trim()) + .map((s) => parseInt(s)) + .filter((n) => !isNaN(n) && n >= 0); +} + export function loadCommaDelimitToStringArray(str: string): string[] { try { return str.split(',').filter((element) => { @@ -1431,3 +1446,86 @@ export async function checkIfAccountExists( return false; } } + +export function getMarketsAndOracleInfosToLoad( + sdkConfig: any, + perpMarketIndicies: number[] | undefined, + spotMarketIndicies: number[] | undefined +): { + oracleInfos: OracleInfo[]; + perpMarketIndicies: number[] | undefined; + spotMarketIndicies: number[] | undefined; +} { + const oracleInfos: OracleInfo[] = []; + const oraclesTracked = new Set(); + + // only watch all markets if neither env vars are specified + const noMarketsSpecified = !perpMarketIndicies && !spotMarketIndicies; + + let perpIndexes = perpMarketIndicies; + if (!perpIndexes) { + if (noMarketsSpecified) { + perpIndexes = sdkConfig.PERP_MARKETS.map( + (m: PerpMarketConfig) => m.marketIndex + ); + } else { + perpIndexes = []; + } + } + let spotIndexes = spotMarketIndicies; + if (!spotIndexes) { + if (noMarketsSpecified) { + spotIndexes = sdkConfig.SPOT_MARKETS.map( + (m: SpotMarketConfig) => m.marketIndex + ); + } else { + spotIndexes = []; + } + } + + if (perpIndexes && perpIndexes.length > 0) { + for (const idx of perpIndexes) { + const perpMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; + if (!perpMarketConfig) { + throw new Error(`Perp market config for ${idx} not found`); + } + const oracleKey = perpMarketConfig.oracle.toBase58(); + if (!oraclesTracked.has(oracleKey)) { + logger.info(`Tracking oracle ${oracleKey} for perp market ${idx}`); + oracleInfos.push({ + publicKey: perpMarketConfig.oracle, + source: perpMarketConfig.oracleSource, + }); + oraclesTracked.add(oracleKey); + } + } + logger.info(`Bot tracking perp markets: ${JSON.stringify(perpIndexes)}`); + } + + if (spotIndexes && spotIndexes.length > 0) { + for (const idx of spotIndexes) { + const spotMarketConfig = sdkConfig.SPOT_MARKETS[idx] as SpotMarketConfig; + if (!spotMarketConfig) { + throw new Error(`Spot market config for ${idx} not found`); + } + const oracleKey = spotMarketConfig.oracle.toBase58(); + if (!oraclesTracked.has(oracleKey)) { + logger.info(`Tracking oracle ${oracleKey} for spot market ${idx}`); + oracleInfos.push({ + publicKey: spotMarketConfig.oracle, + source: spotMarketConfig.oracleSource, + }); + oraclesTracked.add(oracleKey); + } + } + logger.info(`Bot tracking spot markets: ${JSON.stringify(spotIndexes)}`); + } + + return { + oracleInfos, + perpMarketIndicies: + perpIndexes && perpIndexes.length > 0 ? perpIndexes : undefined, + spotMarketIndicies: + spotIndexes && spotIndexes.length > 0 ? spotIndexes : undefined, + }; +} From 2a5547d53a75222dfecf5d911ba8d4d23025dc25 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 12 Nov 2024 10:59:43 -0800 Subject: [PATCH 3/5] change up taker example --- src/experimental-bots/swift/takerExample.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/experimental-bots/swift/takerExample.ts b/src/experimental-bots/swift/takerExample.ts index 3c9d1479..874e3c3b 100644 --- a/src/experimental-bots/swift/takerExample.ts +++ b/src/experimental-bots/swift/takerExample.ts @@ -9,6 +9,7 @@ import { } from '@drift-labs/sdk'; import { RuntimeSpec } from 'src/metrics'; import * as axios from 'axios'; +import { sleepMs } from 'src/utils'; const CONFIRM_TIMEOUT = 30_000; @@ -34,18 +35,24 @@ export class SwiftTaker { } async startInterval() { + const marketIndexes = [0, 1, 2, 3, 4, 5, 6]; this.interval = setInterval(async () => { + await sleepMs(Math.random() * 1000); // Randomize for different grafana metrics const slot = await this.driftClient.connection.getSlot(); const direction = Math.random() > 0.5 ? PositionDirection.LONG : PositionDirection.SHORT; - const oracleInfo = this.driftClient.getOracleDataForPerpMarket(0); - const highPrice = oracleInfo.price.muln(102).divn(100); - const lowPrice = oracleInfo.price.muln(101).divn(100); + const marketIndex = + marketIndexes[Math.floor(Math.random() * marketIndexes.length)]; + + const oracleInfo = + this.driftClient.getOracleDataForPerpMarket(marketIndex); + const highPrice = oracleInfo.price.muln(101).divn(100); + const lowPrice = oracleInfo.price; const orderMessage = { swiftOrderParams: getMarketOrderParams({ - marketIndex: 0, + marketIndex, marketType: MarketType.PERP, direction, baseAssetAmount: BASE_PRECISION, @@ -53,7 +60,7 @@ export class SwiftTaker { ? lowPrice : highPrice, auctionEndPrice: isVariant(direction, 'long') ? highPrice : lowPrice, - auctionDuration: 100, + auctionDuration: 30, }), subAccountId: 0, stopLossOrderParams: null, From b38d540bf9ed0ab2acf5a5552b1c4fff014aa011 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 12 Nov 2024 11:10:01 -0800 Subject: [PATCH 4/5] correct import path --- src/experimental-bots/swift/takerExample.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/experimental-bots/swift/takerExample.ts b/src/experimental-bots/swift/takerExample.ts index 874e3c3b..1911ec80 100644 --- a/src/experimental-bots/swift/takerExample.ts +++ b/src/experimental-bots/swift/takerExample.ts @@ -9,7 +9,7 @@ import { } from '@drift-labs/sdk'; import { RuntimeSpec } from 'src/metrics'; import * as axios from 'axios'; -import { sleepMs } from 'src/utils'; +import { sleepMs } from '../../utils'; const CONFIRM_TIMEOUT = 30_000; From 783417a652b82786d9d4b745bc2b142d50f38553 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 12 Nov 2024 11:21:59 -0800 Subject: [PATCH 5/5] change marketIndex as part of message type --- src/experimental-bots/swift/takerExample.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/experimental-bots/swift/takerExample.ts b/src/experimental-bots/swift/takerExample.ts index 1911ec80..890eb714 100644 --- a/src/experimental-bots/swift/takerExample.ts +++ b/src/experimental-bots/swift/takerExample.ts @@ -77,7 +77,7 @@ export class SwiftTaker { const response = await axios.default.post( 'https://master.swift.drift.trade/orders', { - market_index: 0, + market_index: marketIndex, market_type: 'perp', message: this.driftClient .encodeSwiftOrderParamsMessage(orderMessage)