From 3b9f085da555dba950ca6ee98b3c69158353b8b3 Mon Sep 17 00:00:00 2001 From: Fuxing Loh <4266087+fuxingloh@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:35:22 +0800 Subject: [PATCH] feat(sync): convert sync into a CLI to allow extendability (#162) --- .../{sync-trustwallet-assets.yml => sync.yml} | 32 +++-- .husky/pre-commit | 3 - .idea/frontmatter.iml | 2 +- package.json | 4 +- pnpm-lock.yaml | 5 +- workspace/sync-trustwallet-assets/.gitignore | 1 - .../sync-trustwallet-assets/src/gitlog.ts | 17 --- .../sync-trustwallet-assets/src/index.ts | 59 -------- workspace/sync-trustwallet-assets/src/sync.ts | 118 ---------------- workspace/sync-trustwallet-assets/turbo.json | 9 -- workspace/sync/.gitignore | 1 + .../package.json | 5 +- workspace/sync/src/SyncCommand.ts | 66 +++++++++ workspace/sync/src/bin.ts | 5 + workspace/sync/src/repo/trustwallet/assets.ts | 133 ++++++++++++++++++ .../tsconfig.build.json | 0 .../tsconfig.json | 0 17 files changed, 237 insertions(+), 223 deletions(-) rename .github/workflows/{sync-trustwallet-assets.yml => sync.yml} (57%) delete mode 100644 workspace/sync-trustwallet-assets/.gitignore delete mode 100644 workspace/sync-trustwallet-assets/src/gitlog.ts delete mode 100644 workspace/sync-trustwallet-assets/src/index.ts delete mode 100644 workspace/sync-trustwallet-assets/src/sync.ts delete mode 100644 workspace/sync-trustwallet-assets/turbo.json create mode 100644 workspace/sync/.gitignore rename workspace/{sync-trustwallet-assets => sync}/package.json (83%) create mode 100644 workspace/sync/src/SyncCommand.ts create mode 100644 workspace/sync/src/bin.ts create mode 100644 workspace/sync/src/repo/trustwallet/assets.ts rename workspace/{sync-trustwallet-assets => sync}/tsconfig.build.json (100%) rename workspace/{sync-trustwallet-assets => sync}/tsconfig.json (100%) diff --git a/.github/workflows/sync-trustwallet-assets.yml b/.github/workflows/sync.yml similarity index 57% rename from .github/workflows/sync-trustwallet-assets.yml rename to .github/workflows/sync.yml index 2567d90ba..7a6af28e3 100644 --- a/.github/workflows/sync-trustwallet-assets.yml +++ b/.github/workflows/sync.yml @@ -6,8 +6,8 @@ on: branches: - main paths: - - .github/workflows/sync-trustwallet-assets.yml - - workspace/sync-trustwallet-assets/**/* + - .github/workflows/sync.yml + - workspace/sync/**/* schedule: - cron: '0 0 * * *' @@ -22,6 +22,11 @@ jobs: main: runs-on: ubuntu-latest environment: FRONTMATTER_BOT + strategy: + fail-fast: false + matrix: + repo: + - trustwallet/assets steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -33,14 +38,21 @@ jobs: - run: pnpm install --frozen-lockfile + - run: pnpm turbo run build + working-directory: workspace/sync + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - path: workspace/sync-trustwallet-assets/repo - repository: trustwallet/assets + # PLEASE be aware that this is a security risk when you allow untrusted repositories to be checked out. + # While `pnpm run sync` is executed such that we're only reading files which pose no security risk, + # it's nonetheless possible through other means to execute arbitrary code + # when you attempt to do "TOO MUCH" outside its intended use case. + path: workspace/sync/repo + repository: ${{ matrix.repo }} ref: master - - run: pnpm turbo run sync - working-directory: workspace/sync-trustwallet-assets + - run: pnpm run sync ${{ matrix.repo }} + working-directory: workspace/sync - id: app uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 @@ -51,13 +63,13 @@ jobs: - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 with: token: ${{ steps.app.outputs.token }} - commit-message: 'chore(sync): trustwallet/assets' - title: 'chore(sync): trustwallet/assets' + commit-message: 'chore(sync): ${{ matrix.repo }}' + title: 'chore(sync): ${{ matrix.repo }}' committer: Frontmatter Bot author: Frontmatter Bot body: | #### What this PR does / why we need it: - Sync latest changes from `trustwallet/assets` repository. + Sync latest changes from `${{ matrix.repo }}` repository using `@workspace/sync`. - branch: sync/trustwallet/assets + branch: sync/${{ matrix.repo }} diff --git a/.husky/pre-commit b/.husky/pre-commit index d24fdfc60..2312dc587 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" - npx lint-staged diff --git a/.idea/frontmatter.iml b/.idea/frontmatter.iml index f4c2c5c50..91314b4e4 100644 --- a/.idea/frontmatter.iml +++ b/.idea/frontmatter.iml @@ -16,13 +16,13 @@ - + diff --git a/package.json b/package.json index ef44469c2..f517a7276 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "clean": "turbo run clean", "format": "prettier --write .", "lint": "turbo run lint -- --fix", - "prepare": "husky install", + "prepare": "husky", "test": "turbo run test" }, "lint-staged": { @@ -27,7 +27,7 @@ "turbo": "^1.11.3", "typescript": "5.3.3" }, - "packageManager": "pnpm@8.14.3", + "packageManager": "pnpm@8.15.0", "engines": { "node": "^18 <19", "pnpm": "^8 <9" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f46fef249..583dc5193 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -400,8 +400,11 @@ importers: specifier: ^0.5.11 version: 0.5.11(prettier@3.2.4) - workspace/sync-trustwallet-assets: + workspace/sync: dependencies: + clipanion: + specifier: 3.2.1 + version: 3.2.1(typanion@3.14.0) yaml: specifier: ^2.3.4 version: 2.3.4 diff --git a/workspace/sync-trustwallet-assets/.gitignore b/workspace/sync-trustwallet-assets/.gitignore deleted file mode 100644 index 55d62c108..000000000 --- a/workspace/sync-trustwallet-assets/.gitignore +++ /dev/null @@ -1 +0,0 @@ -repo/ \ No newline at end of file diff --git a/workspace/sync-trustwallet-assets/src/gitlog.ts b/workspace/sync-trustwallet-assets/src/gitlog.ts deleted file mode 100644 index 6b4673e8a..000000000 --- a/workspace/sync-trustwallet-assets/src/gitlog.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { exec } from 'node:child_process'; - -export function getAuthorName(filepath: string): Promise { - return new Promise((resolve, reject) => { - exec(`git log -1 --pretty=format:"%ae" ${filepath}`, (error, stdout, stderr) => { - if (error) { - reject(error); - return; - } - if (stderr) { - reject(stderr); - return; - } - resolve(stdout); - }); - }); -} diff --git a/workspace/sync-trustwallet-assets/src/index.ts b/workspace/sync-trustwallet-assets/src/index.ts deleted file mode 100644 index 7fa11c83a..000000000 --- a/workspace/sync-trustwallet-assets/src/index.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { join } from 'node:path'; -import process from 'node:process'; - -import { DefaultSync } from './sync'; - -export async function sync(): Promise { - const cwd = process.cwd(); - - await new DefaultSync( - join(cwd, 'repo/blockchains/ethereum/assets'), - join(cwd, '../../packages/eip155-1/frontmatter/erc20'), - ).sync((info) => info.type === 'ERC20'); - await new DefaultSync( - join(cwd, 'repo/blockchains/polygon/assets'), - join(cwd, '../../packages/eip155-137/frontmatter/erc20'), - ).sync((info) => info.type === 'POLYGON'); - await new DefaultSync( - join(cwd, 'repo/blockchains/avalanchec/assets'), - join(cwd, '../../packages/eip155-43114/frontmatter/erc20'), - ).sync((info) => info.type === 'AVALANCHE'); - await new DefaultSync( - join(cwd, 'repo/blockchains/smartchain/assets'), - join(cwd, '../../packages/eip155-56/frontmatter/erc20'), - ).sync((info) => info.type === 'BEP20'); - await new DefaultSync( - join(cwd, 'repo/blockchains/arbitrum/assets'), - join(cwd, '../../packages/eip155-42161/frontmatter/erc20'), - ).sync((info) => info.type === 'ARBITRUM'); - await new DefaultSync( - join(cwd, 'repo/blockchains/optimism/assets'), - join(cwd, '../../packages/eip155-10/frontmatter/erc20'), - ).sync((info) => info.type === 'OPTIMISM'); - await new DefaultSync( - join(cwd, 'repo/blockchains/aurora/assets'), - join(cwd, '../../packages/eip155-1313161554/frontmatter/erc20'), - ).sync((info) => info.type === 'AURORA'); - await new DefaultSync( - join(cwd, 'repo/blockchains/celo/assets'), - join(cwd, '../../packages/eip155-42220/frontmatter/erc20'), - ).sync((info) => info.type === 'CELO'); - await new DefaultSync( - join(cwd, 'repo/blockchains/base/assets'), - join(cwd, '../../packages/eip155-8453/frontmatter/erc20'), - ).sync((info) => info.type === 'BASE'); - await new DefaultSync( - join(cwd, 'repo/blockchains/tron/assets'), - join(cwd, '../../packages/tip474-728126428/frontmatter/trc10'), - ).sync((info) => info.type === 'TRC10'); - await new DefaultSync( - join(cwd, 'repo/blockchains/tron/assets'), - join(cwd, '../../packages/tip474-728126428/frontmatter/trc20'), - ).sync((info) => info.type === 'TRC20'); - await new DefaultSync( - join(cwd, 'repo/blockchains/solana/assets'), - join(cwd, '../../packages/solana-5eykt4usfv8p8njdtrepy1vzqkqzkvdp/frontmatter/token'), - ).sync((info) => info.type === 'SPL'); -} - -void sync(); diff --git a/workspace/sync-trustwallet-assets/src/sync.ts b/workspace/sync-trustwallet-assets/src/sync.ts deleted file mode 100644 index 7985986ac..000000000 --- a/workspace/sync-trustwallet-assets/src/sync.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { copyFile, mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises'; - -import { stringify } from 'yaml'; - -import { getAuthorName } from './gitlog'; - -interface Info { - name: string; - website: string; - description: string; - explorer: string; - type: string; - symbol: string; - decimals: number; - status: string; - id: string; - tags: string[]; - links: { - name: string; - url: string; - }[]; -} - -export abstract class Sync { - constructor( - protected readonly from: string, - protected readonly to: string, - ) {} - - async sync(filter: (info: Partial) => boolean): Promise { - for (const dir of await readdir(this.from)) { - const info = JSON.parse( - await readFile(`${this.from}/${dir}/info.json`, { - encoding: 'utf-8', - }), - ) as Partial; - - if (!(await this.shouldWrite(dir, info))) continue; - if (!filter(info)) continue; - await this.write(dir, info); - } - } - - async shouldWrite(dir: string, info: Partial): Promise { - // Make sure info.json exists - const hasFromInfo = await hasFile(`${this.from}/${dir}/info.json`); - if (!hasFromInfo) return false; - - // If README.md and icon.png do not exist on the other side, write it - const namespace = this.getNamespace(info); - const hasToIcon = await hasFile(`${this.to}/${namespace}/icon.png`); - const hasToReadme = await hasFile(`${this.to}/${namespace}/README.md`); - if (!hasToIcon || !hasToReadme) return true; - - // Otherwise, allow overwriting if the author is Frontmatter Bot - const name = await getAuthorName(`${this.to}/${namespace}/README.md`); - return name === 'Frontmatter Bot'; - } - - abstract write(dir: string, info: Partial): Promise; - - abstract getNamespace(info: Partial): string; - - createLinks(info: Partial): Info['links'] { - const links: Info['links'] = []; - if (info.website) links.push({ name: 'website', url: info.website }); - if (info.explorer) links.push({ name: 'explorer', url: info.explorer }); - - if (info.links) { - for (const link of info.links) { - if (link.name === 'website' || link.name === 'explorer') continue; - if (!link.url?.startsWith('https://')) continue; - if (!link.name) continue; - links.push(link); - } - } - return links; - } -} - -function hasFile(filepath: string): Promise { - return stat(filepath).then( - () => true, - () => false, - ); -} - -export class DefaultSync extends Sync { - async write(dir: string, info: Partial): Promise { - const namespace = this.getNamespace(info); - await mkdir(`${this.to}/${namespace}`, { recursive: true }); - - if (await hasFile(`${this.from}/${dir}/logo.png`)) { - await copyFile(`${this.from}/${dir}/logo.png`, `${this.to}/${namespace}/icon.png`); - } - await writeFile( - `${this.to}/${namespace}/README.md`, - [ - `---`, - stringify({ - symbol: info.symbol, - decimals: info.decimals, - tags: info.tags, - links: this.createLinks(info), - }), - `---`, - '', - `# ${info.name}`, - '', - info.description !== '-' ? info.description : '', - ].join('\n'), - ); - } - - getNamespace(info: Partial): string { - return info.id!; - } -} diff --git a/workspace/sync-trustwallet-assets/turbo.json b/workspace/sync-trustwallet-assets/turbo.json deleted file mode 100644 index 32a8af71e..000000000 --- a/workspace/sync-trustwallet-assets/turbo.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://turbo.build/schema.json", - "extends": ["//"], - "pipeline": { - "sync": { - "dependsOn": ["build"] - } - } -} diff --git a/workspace/sync/.gitignore b/workspace/sync/.gitignore new file mode 100644 index 000000000..bc33e4cfd --- /dev/null +++ b/workspace/sync/.gitignore @@ -0,0 +1 @@ +/repo \ No newline at end of file diff --git a/workspace/sync-trustwallet-assets/package.json b/workspace/sync/package.json similarity index 83% rename from workspace/sync-trustwallet-assets/package.json rename to workspace/sync/package.json index ea5890fbc..391a51fd2 100644 --- a/workspace/sync-trustwallet-assets/package.json +++ b/workspace/sync/package.json @@ -1,12 +1,12 @@ { - "name": "@workspace/sync-trustwallet-assets", + "name": "@workspace/sync", "version": "0.0.0", "private": true, "scripts": { "build": "tsc --project tsconfig.build.json", "clean": "rm -rf dist", "lint": "eslint .", - "sync": "node dist/index.js" + "sync": "node dist/bin.js" }, "lint-staged": { "*": [ @@ -18,6 +18,7 @@ ] }, "dependencies": { + "clipanion": "3.2.1", "yaml": "^2.3.4" }, "devDependencies": { diff --git a/workspace/sync/src/SyncCommand.ts b/workspace/sync/src/SyncCommand.ts new file mode 100644 index 000000000..9c4120f5e --- /dev/null +++ b/workspace/sync/src/SyncCommand.ts @@ -0,0 +1,66 @@ +import { mkdir, readdir, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { Command } from 'clipanion'; +import { stringify } from 'yaml'; + +export interface README { + frontmatter: { + symbol: string; + decimals: number; + tags: string[]; + links: { + name: string; + url: string; + }[]; + }; + title: string; + body: string; +} + +export abstract class SyncCommand extends Command { + async walkDir( + dir: string, + options: { + toPath: (data: Data) => string; + filter: (data: Data) => boolean; + }, + ): Promise { + for (const entry of await readdir(dir)) { + const fromPath = join(dir, entry); + const data = await this.readData(fromPath); + + if (data === undefined) continue; + if (!options.filter(data)) continue; + + const toPath = options.toPath(data); + if (!(await this.shouldWrite(data, fromPath, toPath))) continue; + await this.write(data, fromPath, toPath); + } + } + + abstract readData(fromPath: string): Promise; + + abstract toREADME(data: Data): README; + + async shouldWrite(data: Data, fromPath: string, toPath: string): Promise { + return !(await hasFile(join(toPath, 'LOCK'))); + } + + async write(data: Data, fromPath: string, toPath: string): Promise { + await mkdir(toPath, { recursive: true }); + const readmeMd = this.toREADME(data); + + await writeFile( + join(toPath, 'README.md'), + [`---`, stringify(readmeMd.frontmatter), `---`, '', `# ${readmeMd.title}`, '', readmeMd.body].join('\n'), + ); + } +} + +export function hasFile(filepath: string): Promise { + return stat(filepath).then( + () => true, + () => false, + ); +} diff --git a/workspace/sync/src/bin.ts b/workspace/sync/src/bin.ts new file mode 100644 index 000000000..408c1f98b --- /dev/null +++ b/workspace/sync/src/bin.ts @@ -0,0 +1,5 @@ +import { runExit } from 'clipanion'; + +import { TrustWalletAssets } from './repo/trustwallet/assets'; + +void runExit([TrustWalletAssets]); diff --git a/workspace/sync/src/repo/trustwallet/assets.ts b/workspace/sync/src/repo/trustwallet/assets.ts new file mode 100644 index 000000000..9c411e394 --- /dev/null +++ b/workspace/sync/src/repo/trustwallet/assets.ts @@ -0,0 +1,133 @@ +import { copyFile, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { hasFile, README, SyncCommand } from '../../SyncCommand'; + +interface Info { + name: string; + website: string; + description: string; + explorer: string; + type: string; + symbol: string; + decimals: number; + status: string; + id: string; + tags: string[]; + links: { + name: string; + url: string; + }[]; +} + +export class TrustWalletAssets extends SyncCommand { + static override paths = [[`trustwallet/assets`]]; + + async execute(): Promise { + await this.walkDir('repo/blockchains/ethereum/assets', { + toPath: (data) => `../../packages/eip155-1/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'ERC20', + }); + + await this.walkDir('repo/blockchains/polygon/assets', { + toPath: (data) => `../../packages/eip155-137/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'POLYGON', + }); + + await this.walkDir('repo/blockchains/avalanchec/assets', { + toPath: (data) => `../../packages/eip155-43114/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'AVALANCHE', + }); + + await this.walkDir('repo/blockchains/smartchain/assets', { + toPath: (data) => `../../packages/eip155-56/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'BEP20', + }); + + await this.walkDir('repo/blockchains/arbitrum/assets', { + toPath: (data) => `../../packages/eip155-42161/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'ARBITRUM', + }); + + await this.walkDir('repo/blockchains/optimism/assets', { + toPath: (data) => `../../packages/eip155-10/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'OPTIMISM', + }); + + await this.walkDir('repo/blockchains/aurora/assets', { + toPath: (data) => `../../packages/eip155-1313161554/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'AURORA', + }); + + await this.walkDir('repo/blockchains/celo/assets', { + toPath: (data) => `../../packages/eip155-42220/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'CELO', + }); + + await this.walkDir('repo/blockchains/base/assets', { + toPath: (data) => `../../packages/eip155-8453/frontmatter/erc20/${data.id}`, + filter: (data) => data.type === 'BASE', + }); + + await this.walkDir('repo/blockchains/tron/assets', { + toPath: (data) => `../../packages/tip474-728126428/frontmatter/trc10/${data.id}`, + filter: (data) => data.type === 'TRC10', + }); + + await this.walkDir('repo/blockchains/tron/assets', { + toPath: (data) => `../../packages/tip474-728126428/frontmatter/trc20/${data.id}`, + filter: (data) => data.type === 'TRC20', + }); + + await this.walkDir('repo/blockchains/solana/assets', { + toPath: (data) => `../../packages/solana-5eykt4usfv8p8njdtrepy1vzqkqzkvdp/frontmatter/token/${data.id}`, + filter: (data) => data.type === 'SPL', + }); + } + + async readData(path: string): Promise { + return JSON.parse( + await readFile(join(path, 'info.json'), { + encoding: 'utf-8', + }), + ) as Info; + } + + async write(data: Info, fromPath: string, toPath: string): Promise { + await super.write(data, fromPath, toPath); + + const logoPath = join(fromPath, 'logo.png'); + if (await hasFile(logoPath)) { + await copyFile(logoPath, join(toPath, 'icon.png')); + } + } + + toREADME(data: Info): README { + return { + frontmatter: { + symbol: data.symbol, + decimals: data.decimals, + tags: data.tags, + links: createLinks(data), + }, + title: data.name, + body: data.description !== '-' ? data.description : '', + }; + } +} + +function createLinks(info: Partial): README['frontmatter']['links'] { + const links: Info['links'] = []; + if (info.website) links.push({ name: 'website', url: info.website }); + if (info.explorer) links.push({ name: 'explorer', url: info.explorer }); + + if (info.links) { + for (const link of info.links) { + if (link.name === 'website' || link.name === 'explorer') continue; + if (!link.url?.startsWith('https://')) continue; + if (!link.name) continue; + links.push(link); + } + } + return links; +} diff --git a/workspace/sync-trustwallet-assets/tsconfig.build.json b/workspace/sync/tsconfig.build.json similarity index 100% rename from workspace/sync-trustwallet-assets/tsconfig.build.json rename to workspace/sync/tsconfig.build.json diff --git a/workspace/sync-trustwallet-assets/tsconfig.json b/workspace/sync/tsconfig.json similarity index 100% rename from workspace/sync-trustwallet-assets/tsconfig.json rename to workspace/sync/tsconfig.json