From cb4b333ccad792914cfd03d13a8b6547f369dafa Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 12 Apr 2024 12:55:25 -0700 Subject: [PATCH] chore: remove legacy upload codepath plus feature flag and related tests --- decisions/20240313-try-w3up.md | 8 - packages/api/src/bindings.d.ts | 8 - packages/api/src/config.js | 3 - packages/api/src/routes/nfts-upload.js | 51 +---- packages/api/src/utils/context.js | 2 - packages/api/test/nfts-get.spec.js | 3 - packages/api/test/nfts-upload.spec.js | 270 +------------------------ 7 files changed, 4 insertions(+), 341 deletions(-) diff --git a/decisions/20240313-try-w3up.md b/decisions/20240313-try-w3up.md index fdc1e3af3a..d055e5667b 100644 --- a/decisions/20240313-try-w3up.md +++ b/decisions/20240313-try-w3up.md @@ -86,14 +86,6 @@ configures how nft.storage will authenticate to web3.storage when sending invoca configures the capabilities that nft.storage has access to when interacting with web3.storage to store nfts. These capabilities will usually be UCAN delegations whose audience is the identifier of `W3_NFTSTORAGE_PRINCIPAL`. W3_NFTSTORAGE_PROOF needs to have proof rooted in W3_NFTSTORAGE_SPACE that authorize W3_NFTSTORAGE_PRINCIPAL to store in W3_NFTSTORAGE_SPACE. -##### `W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS` Environment Variable - -configures feature switch for which nftstorage accounts will have new uploads stored in web3.storage. - -Note: this environment variable may not be a permanent addition to the codebase. It's only meant to be used as a feature switch that decouples enabling the new functionality from deploying the new code. After testing, we may remove or change this feature switch when it is no longer useful. - -Format: JSON Array of email address strings. - #### UI Changes None. But the existing UI workflow of uploading via https://nft.storage/files/ and form should behave just like they do now. But after this change, there should be a new side effect, which is that the upload should appear in the listing of uploads for the configured `W3_NFTSTORAGE_SPACE` (e.g. via w3cli `w3 ls` or in console.web3.storage). diff --git a/packages/api/src/bindings.d.ts b/packages/api/src/bindings.d.ts index 31ba46662d..248ac684f9 100644 --- a/packages/api/src/bindings.d.ts +++ b/packages/api/src/bindings.d.ts @@ -113,13 +113,6 @@ export interface ServiceConfiguration { /** did:key of the w3up space in which to store NFTs */ W3_NFTSTORAGE_SPACE?: string - - /** - * JSON array of strings that are emails whose uploads should be uploaded via w3up. - * This is meant as a feature switch to test new functionality, - * and this configuration may be removed once the feature switch isn't needed to limit access. - */ - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string } export interface Ucan { @@ -161,7 +154,6 @@ export interface RouteContext { W3_NFTSTORAGE_PRINCIPAL?: string W3_NFTSTORAGE_PROOF?: string W3_NFTSTORAGE_SPACE?: string - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string w3up?: W3upClient contentClaims?: ContentClaimsClient } diff --git a/packages/api/src/config.js b/packages/api/src/config.js index a835b7afde..50603f5259 100644 --- a/packages/api/src/config.js +++ b/packages/api/src/config.js @@ -67,8 +67,6 @@ export function serviceConfigFromVariables(vars) { W3_NFTSTORAGE_PRINCIPAL: vars.W3_NFTSTORAGE_PRINCIPAL, W3_NFTSTORAGE_PROOF: vars.W3_NFTSTORAGE_PROOF, W3_NFTSTORAGE_SPACE: vars.W3_NFTSTORAGE_SPACE, - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: - vars.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS, } } @@ -136,7 +134,6 @@ export function loadConfigVariables() { 'W3_NFTSTORAGE_SPACE', 'W3_NFTSTORAGE_PRINCIPAL', 'W3_NFTSTORAGE_PROOF', - 'W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS', ] for (const name of optional) { diff --git a/packages/api/src/routes/nfts-upload.js b/packages/api/src/routes/nfts-upload.js index 6f36eaea09..5c7de14930 100644 --- a/packages/api/src/routes/nfts-upload.js +++ b/packages/api/src/routes/nfts-upload.js @@ -108,24 +108,6 @@ export async function nftUpload(event, ctx) { return new JSONResponse({ ok: true, value: toNFTResponse(upload) }) } -/** - * returns whether w3up uploading feature is enabled given context + event - * @param {object} context - context of server operation, e.g. including configuration of feature switch - * @param {string} [context.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS] - JSON array of allowed emails - * @param {object} event - specific event for which we should determine whether w3up feature is enabled - * @param {object} event.user - * @param {string} event.user.email - email address of user associated with event - */ -function w3upFeatureSwitchEnabled(context, event) { - // const { W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS = '[]' } = context - // const allowedEmails = JSON.parse(W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS) - // if (!Array.isArray(allowedEmails)) return false - // const eventHasAllowedEmail = allowedEmails.find( - // (allowed) => allowed === event.user.email - // ) - return true -} - /** * @typedef {{ * event: FetchEvent, @@ -170,10 +152,8 @@ export async function uploadCarWithStat( /** @type {(() => Promise)|undefined} */ let checkDagStructureTask const backupUrls = [] - // @ts-expect-error email is not expected in types - if (ctx.w3up && w3upFeatureSwitchEnabled(ctx, { user })) { - const { w3up } = ctx - + const { w3up } = ctx + if (w3up) { // we perform store/add and upload/add concurrently to save time. await Promise.all([ w3up.capability.store.add(car), @@ -204,32 +184,7 @@ export async function uploadCarWithStat( } } } else { - const carBytes = new Uint8Array(await car.arrayBuffer()) - const [s3Backup, r2Backup] = await Promise.all([ - ctx.s3Uploader.uploadCar(carBytes, stat.cid, user.id, metadata), - ctx.r2Uploader.uploadCar(carBytes, stat.cid, user.id, metadata), - ]) - backupUrls.push(s3Backup.url, r2Backup.url) - - // no need to ask linkdex if it's Complete or Unknown - if (stat.structure === 'Partial') { - // ask linkdex for the dag structure across the set of CARs in S3 for this upload. - checkDagStructureTask = async () => { - try { - const structure = await ctx.linkdexApi.getDagStructure(s3Backup.key) - if (structure === 'Complete') { - return ctx.db.updatePinStatus( - upload.content_cid, - elasticPin(structure) - ) - } - } catch (/** @type {any} */ err) { - if (err.code !== MissingApiUrlCode) { - throw err - } - } - } - } + throw new Error('w3up not defined, cannot upload') } const xName = event.request.headers.get('x-name') let name = xName && decodeURIComponent(xName) diff --git a/packages/api/src/utils/context.js b/packages/api/src/utils/context.js index bd4f4726ad..e387bf4240 100644 --- a/packages/api/src/utils/context.js +++ b/packages/api/src/utils/context.js @@ -73,8 +73,6 @@ export async function getContext(event, params) { W3_NFTSTORAGE_PRINCIPAL: config.W3_NFTSTORAGE_PRINCIPAL, W3_NFTSTORAGE_PROOF: config.W3_NFTSTORAGE_PROOF, W3_NFTSTORAGE_SPACE: config.W3_NFTSTORAGE_SPACE, - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: - config.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS, } let w3up if ( diff --git a/packages/api/test/nfts-get.spec.js b/packages/api/test/nfts-get.spec.js index 151ee62ad2..9c0231cd9c 100644 --- a/packages/api/test/nfts-get.spec.js +++ b/packages/api/test/nfts-get.spec.js @@ -109,9 +109,6 @@ test.before(async (t) => { }) ) ).toString(base64), - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([ - nftStorageAccountEmailAllowListedForW3up, - ]), }, }) }) diff --git a/packages/api/test/nfts-upload.spec.js b/packages/api/test/nfts-upload.spec.js index a187266843..acebfc5504 100644 --- a/packages/api/test/nfts-upload.spec.js +++ b/packages/api/test/nfts-upload.spec.js @@ -88,9 +88,6 @@ test.before(async (t) => { }) ) ).toString(base64), - W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([ - nftStorageAccountEmailAllowListedForW3up, - ]), }, }) }) @@ -131,10 +128,7 @@ test.serial('should upload a single file', async (t) => { test.serial('should forward uploads to W3UP_URL', async (t) => { const initialW3upStoreAddCount = mockW3upStoreAddCount const initialW3upUploadAddCount = mockW3upUploadAddCount - const client = await createClientWithUser(t, { - // note this email should be in W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS env var - email: nftStorageAccountEmailAllowListedForW3up, - }) + const client = await createClientWithUser(t) const mf = getMiniflareContext(t) const file = new Blob(['hello world!'], { type: 'application/text' }) const res = await mf.dispatchFetch('http://miniflare.test/upload', { @@ -330,92 +324,6 @@ test.serial('should upload a single CAR file', async (t) => { t.is(data.content.dag_size, 15, 'correct dag size') }) -// TODO verify with @alanshaw that we don't need to do this in the new upload flow -// TODO remove this once we remove legacy uploads -test.skip('should check dag completness with linkdex-api for partial CAR', async (t) => { - const client = await createClientWithUser(t) - const config = getTestServiceConfig(t) - const mf = getMiniflareContext(t) - - const leaf1 = await Block.encode({ - value: pb.prepare({ Data: 'leaf1' }), - codec: pb, - hasher: sha256, - }) - const leaf2 = await Block.encode({ - value: pb.prepare({ Data: 'leaf2' }), - codec: pb, - hasher: sha256, - }) - const parent = await Block.encode({ - value: pb.prepare({ Links: [leaf1.cid, leaf2.cid] }), - codec: pb, - hasher: sha256, - }) - const cid = parent.cid.toString() - const { writer, out } = CarWriter.create(parent.cid) - writer.put(parent) - writer.put(leaf1) - // leave out leaf2 to make patial car - writer.close() - const carBytes = [] - for await (const chunk of out) { - carBytes.push(chunk) - } - const body = new Blob(carBytes) - - if (!config.LINKDEX_URL) { - throw new Error('LINDEX_URL should be set in test config') - } - - const linkdexMock = getLinkdexMock(t) - mockLinkdexResponse(linkdexMock, 'Complete') - - const res = await mf.dispatchFetch('http://miniflare.test/upload', { - method: 'POST', - headers: { - Authorization: `Bearer ${client.token}`, - 'Content-Type': 'application/car', - }, - body, - }) - - t.truthy(res, 'Server responded') - t.true(res.ok, 'Server response ok') - const { ok, value } = await res.json() - t.truthy(ok, 'Server response payload has `ok` property') - t.is(value.cid, cid, 'Server responded with expected CID') - t.is(value.type, 'application/car', 'type should match blob mime-type') - - const db = getRawClient(config) - - const { data: upload } = await db - .from('upload') - .select('*') - .match({ source_cid: cid, user_id: client.userId }) - .single() - - // @ts-ignore - t.is(upload.source_cid, cid) - t.is(upload.deleted_at, null) - - // wait for the call to mock linkdex-api to complete - await res.waitUntil() - const { data: pin } = await db - .from('pin') - .select('*') - .match({ content_cid: cid, service: 'ElasticIpfs' }) - .single() - - t.is( - pin.status, - 'Pinned', - "Status should be pinned when linkdex-api returns 'Complete'" - ) - t.is(pin.service, 'ElasticIpfs') - t.is(pin.status, 'Pinned') -}) - test.serial('should allow a CAR with unsupported hash function', async (t) => { const client = await createClientWithUser(t) const mf = getMiniflareContext(t) @@ -599,111 +507,6 @@ test.serial('should upload to elastic ipfs', async (t) => { t.is(data.content.pin[0].service, 'ElasticIpfs') }) -// TODO: remove once we have fully removed legacy upload path -test.skip('should create S3 & R2 backups', async (t) => { - const client = await createClientWithUser(t) - const config = getTestServiceConfig(t) - const mf = getMiniflareContext(t) - const { root, car } = await packToBlob({ - input: [{ path: 'test.txt', content: 'S3 backup' }], - }) - - const res = await mf.dispatchFetch('http://miniflare.test/upload', { - method: 'POST', - headers: { Authorization: `Bearer ${client.token}` }, - body: car, - }) - - const { value } = await res.json() - t.is(root.toString(), value.cid) - - const upload = await client.client.getUpload(value.cid, client.userId) - t.truthy(upload) - t.truthy(upload?.backup_urls) - const backup_urls = upload?.backup_urls || [] - - // construct the expected backup URL - const carBuf = await car.arrayBuffer() - const carHash = await getHash(new Uint8Array(carBuf)) - const carCid = await getCarCid(new Uint8Array(carBuf)) - - t.is( - backup_urls[0], - expectedS3BackupUrl(config, root, client.userId, carHash) - ) - t.is(backup_urls[1], expectedR2BackupUrl(config, carCid)) -}) - -// TODO: remove once legacy codepath is fully removed -test.skip('should backup chunked uploads, preserving backup_urls for each chunk', async (t) => { - t.timeout(10_000) - const client = await createClientWithUser(t) - const config = getTestServiceConfig(t) - const mf = getMiniflareContext(t) - const chunkSize = 1024 - const nChunks = 5 - - const files = [] - for (let i = 0; i < nChunks; i++) { - files.push({ - path: `/dir/file-${i}.bin`, - content: getRandomBytes(chunkSize), - }) - } - - const { root, car } = await packToBlob({ - input: files, - maxChunkSize: chunkSize, - }) - const splitter = await TreewalkCarSplitter.fromBlob(car, chunkSize) - const linkdexMock = getLinkdexMock(t) - // respond with 'Partial' 5 times, then 'Complete' once. - mockLinkdexResponse(linkdexMock, 'Partial', 5) - mockLinkdexResponse(linkdexMock, 'Complete', 1) - - const backupUrls = [] - for await (const chunk of splitter.cars()) { - const carParts = [] - for await (const part of chunk) { - carParts.push(part) - } - const carFile = new Blob(carParts, { type: 'application/car' }) - const res = await mf.dispatchFetch('http://miniflare.test/upload', { - method: 'POST', - headers: { Authorization: `Bearer ${client.token}` }, - body: carFile, - }) - - const { value } = await res.json() - t.is(root.toString(), value.cid) - const carCid = await getCarCid(new Uint8Array(await carFile.arrayBuffer())) - const carHash = await getHash(new Uint8Array(await carFile.arrayBuffer())) - backupUrls.push(expectedS3BackupUrl(config, root, client.userId, carHash)) - backupUrls.push(expectedR2BackupUrl(config, carCid)) - } - - const upload = await client.client.getUpload(root.toString(), client.userId) - t.truthy(upload) - t.truthy(upload?.backup_urls) - const backup_urls = upload?.backup_urls || [] - t.truthy(backup_urls.length >= nChunks) // using >= to account for CAR / UnixFS overhead - t.is( - backup_urls.length, - backupUrls.length, - `expected ${backupUrls.length} backup urls, got: ${backup_urls.length}` - ) - - /** @type string[] */ - // @ts-expect-error upload.backup_urls has type unknown[], but it's really string[] - const resultUrls = upload.backup_urls - for (const url of resultUrls) { - t.true( - backupUrls.includes(url), - `upload is missing expected backup url ${url}` - ) - } -}) - test.serial('should upload a single file using ucan', async (t) => { const client = await createClientWithUser(t) const config = getTestServiceConfig(t) @@ -853,77 +656,6 @@ test.serial('should update a single file', async (t) => { t.is(uploadData.name, name) }) -// TODO: remove once legacy upload flow is fully removed -test.skip('should write satnav index', async (t) => { - const client = await createClientWithUser(t) - const config = getTestServiceConfig(t) - const mf = getMiniflareContext(t) - const { root, car: carBody } = await createCar('satnav') - const carBytes = new Uint8Array(await carBody.arrayBuffer()) - const carCid = await createCarCid(carBytes) - - const res = await mf.dispatchFetch('http://miniflare.test/upload', { - method: 'POST', - headers: { - Authorization: `Bearer ${client.token}`, - 'Content-Type': 'application/car', - }, - body: carBody, - }) - - const { ok, value } = await res.json() - t.truthy(ok, 'Server response payload has `ok` property') - t.is(value.cid, root.toString(), 'Server responded with expected CID') - t.is(value.type, 'application/car', 'type should match car mime-type') - - const r2Bucket = await mf.getR2Bucket('SATNAV') - const r2Object = await r2Bucket.get(`${carCid}/${carCid}.car.idx`) - if (!r2Object?.body) { - t.fail('repsonse stream must exist') - } - // @ts-expect-error - const reader = MultihashIndexSortedReader.fromIterable(r2Object?.body) - const entries = [] - for await (const entry of reader.entries()) { - entries.push(entry) - } - - t.is(entries.length, 1, 'Index contains a single entry') - t.true( - uint8ArrayEquals(entries[0].digest, root.multihash.digest), - 'Index entry is for root data CID' - ) -}) - -// TODO remove once legacy upload path is removed -test.skip('should write dudewhere index', async (t) => { - const client = await createClientWithUser(t) - const config = getTestServiceConfig(t) - const mf = getMiniflareContext(t) - const { root, car: carBody } = await createCar('dude') - const carBytes = new Uint8Array(await carBody.arrayBuffer()) - const carCid = await createCarCid(carBytes) - - const res = await mf.dispatchFetch('http://miniflare.test/upload', { - method: 'POST', - headers: { - Authorization: `Bearer ${client.token}`, - 'Content-Type': 'application/car', - }, - body: carBody, - }) - - const { ok, value } = await res.json() - t.truthy(ok, 'Server response payload has `ok` property') - t.is(value.cid, root.toString(), 'Server responded with expected CID') - t.is(value.type, 'application/car', 'type should match car mime-type') - - const r2Bucket = await mf.getR2Bucket('DUDEWHERE') - const r2Objects = await r2Bucket.list({ prefix: `${root}/` }) - t.is(r2Objects.objects.length, 1) - t.is(r2Objects.objects[0].key, `${root}/${carCid}`) -}) - test.serial('should fail upload for corrupt CAR', async (t) => { const client = await createClientWithUser(t) const mf = getMiniflareContext(t)