From 553d81a03127863d3eb146132c1efe7e04a5ba11 Mon Sep 17 00:00:00 2001 From: Altynbek Orumbayev Date: Thu, 31 Oct 2024 00:40:13 +0100 Subject: [PATCH 1/2] fix: filename generation and match timestamp format on utils-py --- docs/code/functions/writeAVMDebugTrace.md | 2 +- src/debugging/writeAVMDebugTrace.spec.ts | 46 ++++++++++++++++- src/debugging/writeAVMDebugTrace.ts | 62 ++++++++++++++++++----- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/docs/code/functions/writeAVMDebugTrace.md b/docs/code/functions/writeAVMDebugTrace.md index c021970..1dff60a 100644 --- a/docs/code/functions/writeAVMDebugTrace.md +++ b/docs/code/functions/writeAVMDebugTrace.md @@ -36,4 +36,4 @@ console.log(`Trace content: ${result.traceContent}`); ## Defined in -[debugging/writeAVMDebugTrace.ts:20](https://github.com/algorandfoundation/algokit-utils-ts-debug/blob/main/src/debugging/writeAVMDebugTrace.ts#L20) +[debugging/writeAVMDebugTrace.ts:65](https://github.com/algorandfoundation/algokit-utils-ts-debug/blob/main/src/debugging/writeAVMDebugTrace.ts#L65) diff --git a/src/debugging/writeAVMDebugTrace.spec.ts b/src/debugging/writeAVMDebugTrace.spec.ts index fb87ad0..c9a1e66 100644 --- a/src/debugging/writeAVMDebugTrace.spec.ts +++ b/src/debugging/writeAVMDebugTrace.spec.ts @@ -2,13 +2,15 @@ import { Config, EventType, performAtomicTransactionComposerSimulate } from '@al import { algorandFixture } from '@algorandfoundation/algokit-utils/testing' import { describe, expect, test } from '@jest/globals' import algosdk, { makeEmptyTransactionSigner } from 'algosdk' +import { SimulateResponse } from 'algosdk/dist/types/client/v2/algod/models/types' import * as fs from 'fs/promises' import * as os from 'os' import * as path from 'path' import { DEBUG_TRACES_DIR } from '../constants' import { registerDebugEventHandlers } from '../index' +import { generateDebugTraceFilename } from './writeAVMDebugTrace' -describe('simulateAndPersistResponse tests', () => { +describe('writeAVMDebugTrace tests', () => { const localnet = algorandFixture() beforeAll(async () => { @@ -45,3 +47,45 @@ describe('simulateAndPersistResponse tests', () => { jest.restoreAllMocks() }) }) + +describe('generateDebugTraceFilename', () => { + const TEST_CASES: Array<[string, object, string]> = [ + [ + 'single payment transaction', + { + lastRound: 1000, + txnGroups: [ + { + txnResults: [{ txnResult: { txn: { txn: { type: 'pay' } } } }], + }, + ], + }, + '1pay', + ], + [ + 'multiple transaction types', + { + lastRound: 1000, + txnGroups: [ + { + txnResults: [ + { txnResult: { txn: { txn: { type: 'pay' } } } }, + { txnResult: { txn: { txn: { type: 'pay' } } } }, + { txnResult: { txn: { txn: { type: 'axfer' } } } }, + { txnResult: { txn: { txn: { type: 'appl' } } } }, + { txnResult: { txn: { txn: { type: 'appl' } } } }, + { txnResult: { txn: { txn: { type: 'appl' } } } }, + ], + }, + ], + }, + '2pay_1axfer_3appl', + ], + ] + + test.each(TEST_CASES)('%s', (testName, mockResponse, expectedPattern) => { + const timestamp = '20230101_120000' + const filename = generateDebugTraceFilename(mockResponse as SimulateResponse, timestamp) + expect(filename).toBe(`${timestamp}_lr${(mockResponse as SimulateResponse).lastRound}_${expectedPattern}.trace.avm.json`) + }) +}) diff --git a/src/debugging/writeAVMDebugTrace.ts b/src/debugging/writeAVMDebugTrace.ts index 2c9a899..697619c 100644 --- a/src/debugging/writeAVMDebugTrace.ts +++ b/src/debugging/writeAVMDebugTrace.ts @@ -1,7 +1,52 @@ import { AVMTracesEventData } from '@algorandfoundation/algokit-utils' +import { SimulateResponse } from 'algosdk/dist/types/client/v2/algod/models/types' import { DEBUG_TRACES_DIR } from '../constants' import { getProjectRoot, joinPaths, writeToFile } from '../utils' +type TxnTypeCount = { + type: string + count: number +} + +/** + * Formats a date to YYYYMMDD_HHMMSS in UTC, equivalent to algokit-utils-py format: + * datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") + */ +export function formatTimestampUTC(date: Date): string { + // Get UTC components + const year = date.getUTCFullYear() + const month = String(date.getUTCMonth() + 1).padStart(2, '0') // Months are zero-based + const day = String(date.getUTCDate()).padStart(2, '0') + const hours = String(date.getUTCHours()).padStart(2, '0') + const minutes = String(date.getUTCMinutes()).padStart(2, '0') + const seconds = String(date.getUTCSeconds()).padStart(2, '0') + + // Format the datetime string + return `${year}${month}${day}_${hours}${minutes}${seconds}` +} + +export function generateDebugTraceFilename(simulateResponse: SimulateResponse, timestamp: string): string { + const txnGroups = simulateResponse.txnGroups + const txnTypesCount = txnGroups.reduce((acc: Map, txnGroup) => { + txnGroup.txnResults.forEach(({ txnResult }) => { + const { type } = txnResult.txn.txn + if (!acc.has(type)) { + acc.set(type, { type, count: 0 }) + } + const entry = acc.get(type)! + entry.count++ + }) + return acc + }, new Map()) + + const txnTypesStr = Array.from(txnTypesCount.values()) + .map(({ count, type }) => `${count}${type}`) + .join('_') + + const lastRound = simulateResponse.lastRound + return `${timestamp}_lr${lastRound}_${txnTypesStr}.trace.avm.json` +} + /** * Generates an AVM debug trace from the provided simulation response and persists it to a file. * @@ -20,22 +65,11 @@ import { getProjectRoot, joinPaths, writeToFile } from '../utils' export async function writeAVMDebugTrace(input: AVMTracesEventData): Promise { try { const simulateResponse = input.simulateResponse - const txnGroups = simulateResponse.txnGroups const projectRoot = await getProjectRoot() - - const txnTypesCount = txnGroups.reduce((acc: Record, txnGroup) => { - const txnType = txnGroup.txnResults[0].txnResult.txn.txn.type - acc[txnType] = (acc[txnType] || 0) + 1 - return acc - }, {}) - - const txnTypesStr = Object.entries(txnTypesCount) - .map(([type, count]) => `${count}${type}`) - .join('_') - - const timestamp = new Date().toISOString().replace(/[:.]/g, '') + const timestamp = formatTimestampUTC(new Date()) const outputRootDir = joinPaths(projectRoot, DEBUG_TRACES_DIR) - const outputFilePath = joinPaths(outputRootDir, `${timestamp}_${txnTypesStr}.trace.avm.json`) + const filename = generateDebugTraceFilename(simulateResponse, timestamp) + const outputFilePath = joinPaths(outputRootDir, filename) await writeToFile(outputFilePath, JSON.stringify(simulateResponse.get_obj_for_encoding(), null, 2)) } catch (error) { From 4d34297a17f582eb13bb27daaed5cc55db9beeab Mon Sep 17 00:00:00 2001 From: Altynbek Orumbayev Date: Thu, 31 Oct 2024 12:27:21 +0100 Subject: [PATCH 2/2] fix: restore buffer based old file removal --- docs/code/functions/writeAVMDebugTrace.md | 6 ++- src/debugging/writeAVMDebugTrace.spec.ts | 52 ++++++++++++++++++++++- src/debugging/writeAVMDebugTrace.ts | 51 ++++++++++++++++------ src/index.ts | 2 +- src/utils.ts | 41 ++++++++++++++++++ 5 files changed, 134 insertions(+), 18 deletions(-) diff --git a/docs/code/functions/writeAVMDebugTrace.md b/docs/code/functions/writeAVMDebugTrace.md index 1dff60a..e1abdc7 100644 --- a/docs/code/functions/writeAVMDebugTrace.md +++ b/docs/code/functions/writeAVMDebugTrace.md @@ -6,7 +6,7 @@ # Function: writeAVMDebugTrace() -> **writeAVMDebugTrace**(`input`): `Promise`\<`void`\> +> **writeAVMDebugTrace**(`input`, `bufferSizeMb`): `Promise`\<`void`\> Generates an AVM debug trace from the provided simulation response and persists it to a file. @@ -16,6 +16,8 @@ Generates an AVM debug trace from the provided simulation response and persists The AVMTracesEventData containing the simulation response and other relevant information. +• **bufferSizeMb**: `number` + ## Returns `Promise`\<`void`\> @@ -36,4 +38,4 @@ console.log(`Trace content: ${result.traceContent}`); ## Defined in -[debugging/writeAVMDebugTrace.ts:65](https://github.com/algorandfoundation/algokit-utils-ts-debug/blob/main/src/debugging/writeAVMDebugTrace.ts#L65) +[debugging/writeAVMDebugTrace.ts:85](https://github.com/algorandfoundation/algokit-utils-ts-debug/blob/main/src/debugging/writeAVMDebugTrace.ts#L85) diff --git a/src/debugging/writeAVMDebugTrace.spec.ts b/src/debugging/writeAVMDebugTrace.spec.ts index c9a1e66..ccee078 100644 --- a/src/debugging/writeAVMDebugTrace.spec.ts +++ b/src/debugging/writeAVMDebugTrace.spec.ts @@ -8,7 +8,7 @@ import * as os from 'os' import * as path from 'path' import { DEBUG_TRACES_DIR } from '../constants' import { registerDebugEventHandlers } from '../index' -import { generateDebugTraceFilename } from './writeAVMDebugTrace' +import { cleanupOldFiles, generateDebugTraceFilename } from './writeAVMDebugTrace' describe('writeAVMDebugTrace tests', () => { const localnet = algorandFixture() @@ -89,3 +89,53 @@ describe('generateDebugTraceFilename', () => { expect(filename).toBe(`${timestamp}_lr${(mockResponse as SimulateResponse).lastRound}_${expectedPattern}.trace.avm.json`) }) }) + +describe('cleanupOldFiles', () => { + let tempDir: string + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'debug-traces-')) + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + test('removes oldest files when buffer size is exceeded', async () => { + // Create test files with different timestamps and sizes + const testFiles = [ + { name: 'old.json', content: 'a'.repeat(1024 * 1024), mtime: new Date('2023-01-01') }, + { name: 'newer.json', content: 'b'.repeat(1024 * 1024), mtime: new Date('2023-01-02') }, + { name: 'newest.json', content: 'c'.repeat(1024 * 1024), mtime: new Date('2023-01-03') }, + ] + + // Create files with specific timestamps + for (const file of testFiles) { + const filePath = path.join(tempDir, file.name) + await fs.writeFile(filePath, file.content) + await fs.utimes(filePath, file.mtime, file.mtime) + } + + // Set buffer size to 2MB (should remove oldest file) + await cleanupOldFiles(2, tempDir) + + // Check remaining files + const remainingFiles = await fs.readdir(tempDir) + expect(remainingFiles).toHaveLength(2) + expect(remainingFiles).toContain('newer.json') + expect(remainingFiles).toContain('newest.json') + expect(remainingFiles).not.toContain('old.json') + }) + + test('does nothing when total size is within buffer limit', async () => { + const content = 'a'.repeat(512 * 1024) // 512KB + await fs.writeFile(path.join(tempDir, 'file1.json'), content) + await fs.writeFile(path.join(tempDir, 'file2.json'), content) + + // Set buffer size to 2MB (files total 1MB, should not remove anything) + await cleanupOldFiles(2, tempDir) + + const remainingFiles = await fs.readdir(tempDir) + expect(remainingFiles).toHaveLength(2) + }) +}) diff --git a/src/debugging/writeAVMDebugTrace.ts b/src/debugging/writeAVMDebugTrace.ts index 697619c..07d2ad6 100644 --- a/src/debugging/writeAVMDebugTrace.ts +++ b/src/debugging/writeAVMDebugTrace.ts @@ -1,7 +1,7 @@ import { AVMTracesEventData } from '@algorandfoundation/algokit-utils' import { SimulateResponse } from 'algosdk/dist/types/client/v2/algod/models/types' import { DEBUG_TRACES_DIR } from '../constants' -import { getProjectRoot, joinPaths, writeToFile } from '../utils' +import { createDirForFilePathIfNotExists, formatTimestampUTC, getProjectRoot, joinPaths, writeToFile } from '../utils' type TxnTypeCount = { type: string @@ -9,22 +9,42 @@ type TxnTypeCount = { } /** - * Formats a date to YYYYMMDD_HHMMSS in UTC, equivalent to algokit-utils-py format: - * datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") + * Removes old trace files when total size exceeds buffer limit */ -export function formatTimestampUTC(date: Date): string { - // Get UTC components - const year = date.getUTCFullYear() - const month = String(date.getUTCMonth() + 1).padStart(2, '0') // Months are zero-based - const day = String(date.getUTCDate()).padStart(2, '0') - const hours = String(date.getUTCHours()).padStart(2, '0') - const minutes = String(date.getUTCMinutes()).padStart(2, '0') - const seconds = String(date.getUTCSeconds()).padStart(2, '0') +export async function cleanupOldFiles(bufferSizeMb: number, outputRootDir: string): Promise { + const fs = await import('fs') + const path = await import('path') - // Format the datetime string - return `${year}${month}${day}_${hours}${minutes}${seconds}` + let totalSize = ( + await Promise.all( + (await fs.promises.readdir(outputRootDir)).map(async (file) => (await fs.promises.stat(path.join(outputRootDir, file))).size), + ) + ).reduce((a, b) => a + b, 0) + + if (totalSize > bufferSizeMb * 1024 * 1024) { + const files = await fs.promises.readdir(outputRootDir) + const fileStats = await Promise.all( + files.map(async (file) => { + const stats = await fs.promises.stat(path.join(outputRootDir, file)) + return { file, mtime: stats.mtime, size: stats.size } + }), + ) + + // Sort by modification time (oldest first) + fileStats.sort((a, b) => a.mtime.getTime() - b.mtime.getTime()) + + // Remove oldest files until we're under the buffer size + while (totalSize > bufferSizeMb * 1024 * 1024 && fileStats.length > 0) { + const oldestFile = fileStats.shift()! + totalSize -= oldestFile.size + await fs.promises.unlink(path.join(outputRootDir, oldestFile.file)) + } + } } +/** + * Generates a descriptive filename for a debug trace based on transaction types + */ export function generateDebugTraceFilename(simulateResponse: SimulateResponse, timestamp: string): string { const txnGroups = simulateResponse.txnGroups const txnTypesCount = txnGroups.reduce((acc: Map, txnGroup) => { @@ -62,7 +82,7 @@ export function generateDebugTraceFilename(simulateResponse: SimulateResponse, t * console.log(`Debug trace saved to: ${result.outputPath}`); * console.log(`Trace content: ${result.traceContent}`); */ -export async function writeAVMDebugTrace(input: AVMTracesEventData): Promise { +export async function writeAVMDebugTrace(input: AVMTracesEventData, bufferSizeMb: number): Promise { try { const simulateResponse = input.simulateResponse const projectRoot = await getProjectRoot() @@ -71,6 +91,9 @@ export async function writeAVMDebugTrace(input: AVMTracesEventData): Promise { Config.events.on(EventType.TxnGroupSimulated, async (eventData: AVMTracesEventData) => { - await writeAVMDebugTrace(eventData) + await writeAVMDebugTrace(eventData, Config.traceBufferSizeMb || 256) }) Config.events.on(EventType.AppCompiled, async (data: TealSourcesDebugEventData) => { await writeTealDebugSourceMaps(data) diff --git a/src/utils.ts b/src/utils.ts index 9e89343..5885209 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,13 @@ import { Config } from '@algorandfoundation/algokit-utils' import { DEFAULT_MAX_SEARCH_DEPTH } from './constants' +interface ErrnoException extends Error { + errno?: number + code?: string + path?: string + syscall?: string +} + export const isNode = () => { return typeof process !== 'undefined' && process.versions != null && process.versions.node != null } @@ -13,6 +20,23 @@ export async function writeToFile(filePath: string, content: string): Promise { + const path = await import('path') + const fs = await import('fs') + + try { + await fs.promises.access(path.dirname(filePath)) + } catch (error: unknown) { + const err = error as ErrnoException + + if (err.code === 'ENOENT') { + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }) + } else { + throw err + } + } +} + export async function getProjectRoot(): Promise { const projectRoot = Config.projectRoot @@ -52,3 +76,20 @@ export function joinPaths(...parts: string[]): string { const separator = typeof process !== 'undefined' && process.platform === 'win32' ? '\\' : '/' return parts.join(separator).replace(/\/+/g, separator) } + +/** + * Formats a date to YYYYMMDD_HHMMSS in UTC, equivalent to algokit-utils-py format: + * datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") + */ +export function formatTimestampUTC(date: Date): string { + // Get UTC components + const year = date.getUTCFullYear() + const month = String(date.getUTCMonth() + 1).padStart(2, '0') // Months are zero-based + const day = String(date.getUTCDate()).padStart(2, '0') + const hours = String(date.getUTCHours()).padStart(2, '0') + const minutes = String(date.getUTCMinutes()).padStart(2, '0') + const seconds = String(date.getUTCSeconds()).padStart(2, '0') + + // Format the datetime string + return `${year}${month}${day}_${hours}${minutes}${seconds}` +}