diff --git a/.env.sample b/.env.sample index 440a7de..dba9600 100644 --- a/.env.sample +++ b/.env.sample @@ -8,5 +8,6 @@ ETH_POLYGON_NETWORK=maticmum ETH_RPC=https://example.com/rpc ETH_RPC_MAINNET=https://example.com/rpc ETH_RPC_POLYGON=https://example.com/rpc +KETL_HASHES_SOURCE=https://eth.example.com DOMAIN=verify.sealcred.xyz ENVIRONMENT=development diff --git a/.gitignore b/.gitignore index a1b9b5a..0ac1d8a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ pot/OwnershipChecker_final.zkey !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions + +.node-persist/* diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 78c25ee..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "merkleTrees"] - path = merkleTrees - url = git@github.com:BigWhaleLabs/ketl-merkle-trees.git diff --git a/README.md b/README.md index 92289d9..f107ff0 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ And you should be good to go! Feel free to fork and submit pull requests. | `ETH_RPC_POLYGON` | Polygon node RPC URI (defaults to @bwl/constants) | | `DOMAIN` | Domain name for caddy, DNS should point at the IP where the code is hosted | | `ENVIRONMENT` | Environment name (defaults to `development`) | +| `KETL_HASHES_SOURCE` | Link to merkle tree hashes for Ketl | Also, please, consider looking at `.env.sample`. diff --git a/merkleTrees b/merkleTrees deleted file mode 160000 index 9108984..0000000 --- a/merkleTrees +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 91089849e9439ebfa1e2ddb3ae8fb9c5985f9873 diff --git a/package.json b/package.json index 9fe9688..ecb5fe4 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,11 @@ "@big-whale-labs/seal-cred-email": "^1.1.2", "@hapi/boom": "^10.0.0", "@koa/cors": "^4.0.0", + "@types/node-persist": "^3.1.4", "@zk-kit/incremental-merkle-tree": "^1.0.0", "amala": "^8.0.2", "axios": "^1.5.0", + "axios-cache-interceptor": "^1.3.0", "circomlibjs": "^0.1.7", "dotenv": "^16.0.3", "envalid": "^7.3.1", @@ -33,6 +35,7 @@ "koa": "^2.13.4", "koa-bodyparser": "^4.3.0", "koa-router": "^12.0.0", + "node-persist": "^3.1.3", "nodemailer": "^6.8.0", "uuid": "^9.0.0" }, diff --git a/src/controllers/verify-ketl.ts b/src/controllers/verify-ketl.ts index 06f84ad..066d986 100644 --- a/src/controllers/verify-ketl.ts +++ b/src/controllers/verify-ketl.ts @@ -16,8 +16,9 @@ import Signature from '@/validators/Signature' import Token from '@/validators/Token' import TwitterBody from '@/validators/TwitterBody' import VerificationType from '@/models/VerificationType' +import checkInvite from '@/helpers/ketl/checkInvite' import fetchUserProfile from '@/helpers/twitter/fetchUserProfile' -import getAllowlistMap from '@/helpers/getAllowlistMap' +import getAttestationHash from '@/helpers/signatures/getAttestationHash' import getBalance from '@/helpers/getBalance' import getEmailDomain from '@/helpers/getEmailDomain' import handleInvitationError from '@/helpers/handleInvitationError' @@ -26,21 +27,19 @@ import sendEmail from '@/helpers/sendEmail' import signAttestationMessage from '@/helpers/signatures/signAttestationMessage' import zeroAddress from '@/models/zeroAddress' -const allowlistMap = getAllowlistMap() - @Controller('/verify-ketl') export default class VerifyKetlController { @Version('0.2.2') @Post('/token') - multipleToken( + async multipleToken( @Ctx() ctx: Context, @Body({ required: true }) { token, types }: AttestationTypeList & Token ) { const attestations = [] for (const type of types) { - const allowlist = allowlistMap.get(type) - if (allowlist?.has(`token:${token}`)) - attestations.push(signAttestationMessage(type, hexlifyString(token))) + const attestationHash = await getAttestationHash(hexlifyString(token)) + const record = await signAttestationMessage(type, attestationHash) + if (await checkInvite(type, attestationHash)) attestations.push(record) } if (!attestations.length) @@ -63,13 +62,16 @@ export default class VerifyKetlController { const secret = [] for (const type of types) { - const allowlist = allowlistMap.get(type) - if (!allowlist?.has(`email:${email}`)) continue - const { message, signature } = await signAttestationMessage( - type, + const attestationHash = await getAttestationHash( VerificationType.email, hexlifyString(email) ) + const { message, signature } = await signAttestationMessage( + type, + attestationHash + ) + const hasInvite = await checkInvite(type, attestationHash) + if (!hasInvite) continue if (secret.length === 0) { const attestationHash = message[1] secret.push(attestationHash) @@ -95,13 +97,12 @@ export default class VerifyKetlController { @Body({ required: true }) { email, type }: AttestationType & Email ) { - const { message, signature } = await signAttestationMessage( - type, + const attestationHash = await getAttestationHash( VerificationType.email, hexlifyString(email) ) + const { signature } = await signAttestationMessage(type, attestationHash) const domain = getEmailDomain(email) - const attestationHash = message[1] void sendEmail({ domain, @@ -126,11 +127,13 @@ export default class VerifyKetlController { const attestations = [] for (const type of types) { - const allowlist = allowlistMap.get(type) - if (allowlist?.has(`twitter:${id}`)) - attestations.push( - signAttestationMessage(type, VerificationType.twitter, id) - ) + const attestationHash = await getAttestationHash( + VerificationType.twitter, + id + ) + const record = await signAttestationMessage(type, attestationHash) + const hasInvite = await checkInvite(type, attestationHash) + if (hasInvite) attestations.push(record) } if (!attestations.length) @@ -148,7 +151,11 @@ export default class VerifyKetlController { if (!user) return ctx.throw(badRequest('Failed to fetch user profile')) const { id } = user - return signAttestationMessage(type, VerificationType.twitter, id) + const attestationHash = await getAttestationHash( + VerificationType.twitter, + id + ) + return signAttestationMessage(type, attestationHash) } @Post('/balance-unique') @@ -194,27 +201,17 @@ export default class VerifyKetlController { const attestations = [] for (const type of types) { - const allowlist = allowlistMap.get(type) - if (allowlist?.has(`orangedao:${signerAddress}`)) - attestations.push( - signAttestationMessage( - type, - VerificationType.balance, - hexlifyString(signerAddress), - threshold, - hexlifyString(YC_ALUM_NFT_CONTRACT) - ) - ) - if (allowlist?.has(`bwlnft:${signerAddress}`)) - attestations.push( - signAttestationMessage( - type, - VerificationType.balance, - hexlifyString(signerAddress), - threshold, - hexlifyString(KETL_BWL_NFT_CONTRACT) - ) + for (const contract of [YC_ALUM_NFT_CONTRACT, KETL_BWL_NFT_CONTRACT]) { + const attestationHash = await getAttestationHash( + VerificationType.balance, + hexlifyString(signerAddress), + threshold, + hexlifyString(contract) ) + const record = await signAttestationMessage(type, attestationHash) + const hasInvite = await checkInvite(type, attestationHash) + if (hasInvite) attestations.push(record) + } } if (!attestations.length) return ctx.throw(notFound(handleInvitationError('wallet'))) @@ -258,12 +255,13 @@ export default class VerifyKetlController { return ctx.throw(badRequest("Can't fetch the balances")) } - return signAttestationMessage( - type, + const attestationHash = await getAttestationHash( VerificationType.balance, hexlifyString(ownerAddress.toLowerCase()), threshold, hexlifyString(tokenAddress) ) + + return signAttestationMessage(type, attestationHash) } } diff --git a/src/helpers/axiosWithCache.ts b/src/helpers/axiosWithCache.ts new file mode 100644 index 0000000..a2221f3 --- /dev/null +++ b/src/helpers/axiosWithCache.ts @@ -0,0 +1,7 @@ +import { setupCache } from 'axios-cache-interceptor' +import axios from 'axios' +import buildPersistedStorage from '@/helpers/buildPersistedStorage' + +export default setupCache(axios, { + storage: buildPersistedStorage(), +}) diff --git a/src/helpers/buildPersistedStorage.ts b/src/helpers/buildPersistedStorage.ts new file mode 100644 index 0000000..0405217 --- /dev/null +++ b/src/helpers/buildPersistedStorage.ts @@ -0,0 +1,16 @@ +import { NotEmptyStorageValue, buildStorage } from 'axios-cache-interceptor' +import { getItem, removeItem, setItem } from 'node-persist' + +export default function buildPersistedStorage() { + return buildStorage({ + find(key: string) { + return getItem(key) + }, + async remove(key: string) { + await removeItem(key) + }, + async set(key: string, value: NotEmptyStorageValue) { + await setItem(key, value) + }, + }) +} diff --git a/src/helpers/env.ts b/src/helpers/env.ts index cc6bc91..ba0789e 100644 --- a/src/helpers/env.ts +++ b/src/helpers/env.ts @@ -20,6 +20,10 @@ export default cleanEnv(process.env, { ETH_RPC: str({ default: ETH_RPC }), ETH_RPC_MAINNET: str({ default: ETH_RPC_MAINNET }), ETH_RPC_POLYGON: str(), + KETL_HASHES_SOURCE: str({ + default: + 'https://raw.githubusercontent.com/BigWhaleLabs/ketl-attestation-token/main', + }), PORT: num({ default: 1337 }), SMTP_PASS: str(), SMTP_USER: str(), diff --git a/src/helpers/farcaster/connectedAddresses.ts b/src/helpers/farcaster/connectedAddresses.ts index bffe781..077f8e6 100644 --- a/src/helpers/farcaster/connectedAddresses.ts +++ b/src/helpers/farcaster/connectedAddresses.ts @@ -1,5 +1,7 @@ -import axios from 'axios' +import { isAxiosError } from 'axios' +import axiosWithCache from '@/helpers/axiosWithCache' import checkIfPrimary from '@/helpers/cluster/checkIfPrimary' +import sleep from '@/helpers/sleep' export const faddressToConnectedAddresses = {} as { [faddress: string]: string[] @@ -7,11 +9,13 @@ export const faddressToConnectedAddresses = {} as { export async function fetchConnectedAddress(address: string) { checkIfPrimary() - const { data } = await axios.get< + const { data } = await axiosWithCache.get< { connectedAddress: string }[] - >(`https://searchcaster.xyz/api/profiles?address=${address}`) + >(`https://searchcaster.xyz/api/profiles?address=${address}`, { + id: `farcaster-profiles-${address}`, + }) const connectedAddresses = data.map( ({ connectedAddress }) => connectedAddress ) @@ -36,6 +40,8 @@ export async function fetchConnectedAddresses(addresses: string[]) { 'Error fetching connected addresses', error instanceof Error ? error.message : error ) + if (isAxiosError(error) && error.response?.status === 429) + await sleep(5000) i -= step } } diff --git a/src/helpers/getAllowlistMap.ts b/src/helpers/getAllowlistMap.ts deleted file mode 100644 index 7d40043..0000000 --- a/src/helpers/getAllowlistMap.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as fs from 'fs' -import { resolve } from 'path' -import AttestationType from '@/models/AttestationType' - -function getAllowlist(attestationType: AttestationType) { - try { - const filePath = resolve( - process.cwd(), - 'merkleTrees', - `${attestationType}.txt` - ) - const file = fs.readFileSync(filePath, 'utf8') - const allowlist = file.split('\n') - return allowlist.filter( - (record: string) => !/^#/.test(record) && record !== '' - ) - } catch (e) { - return [] - } -} - -export default function getAllowlistMap(): Map> { - const allowlistMap = new Map() - for (const type in AttestationType) { - const attestationType = Number(type) - if (isNaN(attestationType)) continue - allowlistMap.set(attestationType, new Set(getAllowlist(attestationType))) - } - return allowlistMap -} diff --git a/src/helpers/ketl/checkInvite.ts b/src/helpers/ketl/checkInvite.ts new file mode 100644 index 0000000..f726dfd --- /dev/null +++ b/src/helpers/ketl/checkInvite.ts @@ -0,0 +1,7 @@ +import getHashes from '@/helpers/ketl/getHashes' + +export default async function checkInvite(type: number, hash: string) { + const hashes = await getHashes(type) + + return hashes.has(hash) +} diff --git a/src/helpers/ketl/getHashes.ts b/src/helpers/ketl/getHashes.ts new file mode 100644 index 0000000..2f17b40 --- /dev/null +++ b/src/helpers/ketl/getHashes.ts @@ -0,0 +1,16 @@ +import axiosWithCache from '@/helpers/axiosWithCache' +import env from '@/helpers/env' + +export default async function getHashes(attestationType: number) { + const { data } = await axiosWithCache.get( + `${env.KETL_HASHES_SOURCE}/hashes/${attestationType}.json`, + { + cache: { + ttl: 1000 * 60 * 5, // 5 minute. + }, + id: `hashes-${attestationType}`, + } + ) + + return new Set(data) +} diff --git a/src/helpers/signatures/eddsaSigPoseidon.ts b/src/helpers/signatures/eddsaSigPoseidon.ts index c485320..77d17d4 100644 --- a/src/helpers/signatures/eddsaSigPoseidon.ts +++ b/src/helpers/signatures/eddsaSigPoseidon.ts @@ -6,7 +6,9 @@ import poseidonHash from '@/helpers/signatures/poseidonHash' const privateKey = utils.arrayify(env.EDDSA_PRIVATE_KEY) let eddsa: typeof buildEddsa -export default async function (message: (number | BigNumber)[] | Uint8Array) { +export default async function ( + message: (string | number | BigNumber)[] | Uint8Array +) { const hash = await poseidonHash(message) if (!eddsa) eddsa = await buildEddsa() const signature = eddsa.signPoseidon(privateKey, hash) diff --git a/src/helpers/signatures/getAttestationHash.ts b/src/helpers/signatures/getAttestationHash.ts new file mode 100644 index 0000000..8b7b135 --- /dev/null +++ b/src/helpers/signatures/getAttestationHash.ts @@ -0,0 +1,5 @@ +import poseidonHash from '@/helpers/signatures/poseidonHash' + +export default function getAttestationHash(...attestation: string[]) { + return poseidonHash(attestation, true) +} diff --git a/src/helpers/signatures/signAttestationMessage.ts b/src/helpers/signatures/signAttestationMessage.ts index 5c83b36..f6e9f93 100644 --- a/src/helpers/signatures/signAttestationMessage.ts +++ b/src/helpers/signatures/signAttestationMessage.ts @@ -1,12 +1,10 @@ import AttestationType from '@/models/AttestationType' import eddsaSigPoseidon from '@/helpers/signatures/eddsaSigPoseidon' -import poseidonHash from '@/helpers/signatures/poseidonHash' export default async function signAttestationMessage( attestationType: AttestationType, - ...attestation: string[] + attestationHash: string ) { - const attestationHash = await poseidonHash(attestation, true) const message = [attestationType, attestationHash] const signature = await eddsaSigPoseidon(message) diff --git a/src/helpers/sleep.ts b/src/helpers/sleep.ts new file mode 100644 index 0000000..0179fd0 --- /dev/null +++ b/src/helpers/sleep.ts @@ -0,0 +1,3 @@ +export default function sleep(time = 1000) { + return new Promise((res) => setTimeout(res, time)) +} diff --git a/src/server.ts b/src/server.ts index 5c0371b..94e1faf 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2,14 +2,20 @@ import 'module-alias/register' import 'source-map-support/register' import * as os from 'os' +import { init as initStorage } from 'node-persist' import { isAddressConnected } from '@/helpers/farcaster/connectedAddresses' import Cluster from '@/helpers/cluster/cluster' +import axiosWithCache from '@/helpers/axiosWithCache' +import buildPersistedStorage from '@/helpers/buildPersistedStorage' import prepareFarcaster from '@/helpers/farcaster/prepareFarcaster' import runApp from '@/helpers/runApp' const totalCPUs = os.cpus().length -void (() => { +void (async () => { + await initStorage() + axiosWithCache.storage = buildPersistedStorage() + if (Cluster.isPrimary) { console.log(`Number of CPUs is ${totalCPUs}`) console.log(`Primary ${process.pid} is running`) diff --git a/yarn.lock b/yarn.lock index f4263f9..1cf99b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1281,6 +1281,15 @@ __metadata: languageName: node linkType: hard +"@types/node-persist@npm:^3.1.4": + version: 3.1.4 + resolution: "@types/node-persist@npm:3.1.4" + dependencies: + "@types/node": "*" + checksum: b81a13f37550d9f365b75579b34167cfb7a2d9b0784f43b66c2d5f309f6f547ab31f07bf7d68cbb63adbf8c7a2390db397017e118d29e3dd82b2d6a4122e3151 + languageName: node + linkType: hard + "@types/node@npm:*": version: 18.0.0 resolution: "@types/node@npm:18.0.0" @@ -1725,11 +1734,13 @@ __metadata: "@types/koa-router": ^7.4.4 "@types/koa__cors": ^3.3.0 "@types/node": ^18.11.9 + "@types/node-persist": ^3.1.4 "@types/nodemailer": ^6.4.6 "@types/uuid": ^8.3.4 "@zk-kit/incremental-merkle-tree": ^1.0.0 amala: ^8.0.2 axios: ^1.5.0 + axios-cache-interceptor: ^1.3.0 circomlibjs: ^0.1.7 dotenv: ^16.0.3 envalid: ^7.3.1 @@ -1739,6 +1750,7 @@ __metadata: koa-bodyparser: ^4.3.0 koa-router: ^12.0.0 module-alias: ^2.2.2 + node-persist: ^3.1.3 nodemailer: ^6.8.0 prettier: ^2.8.0 source-map-support: ^0.5.21 @@ -1755,6 +1767,19 @@ __metadata: languageName: node linkType: hard +"axios-cache-interceptor@npm:^1.3.0": + version: 1.3.0 + resolution: "axios-cache-interceptor@npm:1.3.0" + dependencies: + cache-parser: ^1.2.4 + fast-defer: ^1.1.7 + object-code: ^1.3.0 + peerDependencies: + axios: ^1 + checksum: 000138670d97a9bcddf996a4c1b87f186712c9bd4a1ac2ef7a0d49ee1965110923343226596e7419f153945f46ab1aece6fef64aca53836312f3949f4a674aca + languageName: node + linkType: hard + "axios@npm:^1.5.0": version: 1.5.0 resolution: "axios@npm:1.5.0" @@ -1973,6 +1998,13 @@ __metadata: languageName: node linkType: hard +"cache-parser@npm:^1.2.4": + version: 1.2.4 + resolution: "cache-parser@npm:1.2.4" + checksum: de9fc4ab7af318109f1e53474e674d43997bc7b8676157e4f28e7dc60fda2f434ee138aa9d9abb9f849c55e73202ee74865afaa4e00298de70c4326510680c19 + languageName: node + linkType: hard + "call-bind@npm:^1.0.0, call-bind@npm:^1.0.2": version: 1.0.2 resolution: "call-bind@npm:1.0.2" @@ -3251,6 +3283,13 @@ __metadata: languageName: node linkType: hard +"fast-defer@npm:^1.1.7": + version: 1.1.7 + resolution: "fast-defer@npm:1.1.7" + checksum: c0f816fe3f83ca7e12c56a6631c6819a50349fb7e1931cec88e883168b523c65605e6e0df32c8fba6c341860c29ecc6fe2531ba8a418d27d31da6544ad1bf45b + languageName: node + linkType: hard + "fast-diff@npm:^1.1.2": version: 1.2.0 resolution: "fast-diff@npm:1.2.0" @@ -5254,6 +5293,13 @@ __metadata: languageName: node linkType: hard +"node-persist@npm:^3.1.3": + version: 3.1.3 + resolution: "node-persist@npm:3.1.3" + checksum: a67d1f7e646e9926558237b4f30a50cc73ae9e58a1be28fb87203b0b7180b7f7178c380f74229900c62cf7ee139e00c4c8bd5abcbb82e91326d8eb7938a827a9 + languageName: node + linkType: hard + "nodemailer@npm:^6.8.0": version: 6.8.0 resolution: "nodemailer@npm:6.8.0" @@ -5307,6 +5353,13 @@ __metadata: languageName: node linkType: hard +"object-code@npm:^1.3.0": + version: 1.3.0 + resolution: "object-code@npm:1.3.0" + checksum: bc5e3df85ac54785b3d1d9b8a9b2e168eb5f69e02be3c958fc05aebaa85ec659c55903852bf15ebf44e98053212f88eb7d3a9e28a50f10c4d60d820f80aa8d64 + languageName: node + linkType: hard + "object-inspect@npm:^1.12.0, object-inspect@npm:^1.9.0": version: 1.12.2 resolution: "object-inspect@npm:1.12.2"