diff --git a/.github/workflows/w3.yml b/.github/workflows/w3.yml index a1dba46703..0e609c1291 100644 --- a/.github/workflows/w3.yml +++ b/.github/workflows/w3.yml @@ -43,6 +43,9 @@ jobs: # We want this one to fail if we can't upload to the staging api using the workspace version of the client. - uses: bahmutov/npm-install@v1 - name: Test upload to staging + # disabled until we can make this succeed while staging is in maintenance mode + # as part of old.web3.storage sunset + continue-on-error: true run: | npm run build -w packages/client echo "$(date --utc --iso-8601=seconds) web3.storage upload test" > ./upload-test-small diff --git a/package-lock.json b/package-lock.json index b769105abd..9a4c4d762a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24202,7 +24202,7 @@ }, "packages/api": { "name": "@web3-storage/api", - "version": "7.20.0", + "version": "7.22.0", "license": "(Apache-2.0 OR MIT)", "dependencies": { "@aws-sdk/client-s3": "^3.53.1", @@ -25966,6 +25966,7 @@ "license": "(Apache-2.0 OR MIT)", "dependencies": { "@supabase/postgrest-js": "^0.37.0", + "multiformats": "^13.0.0", "p-retry": "^4.6.1" }, "devDependencies": { @@ -25981,6 +25982,11 @@ "standard": "^16.0.3" } }, + "packages/db/node_modules/multiformats": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.0.0.tgz", + "integrity": "sha512-xiIB0p7EKmETm3wyKedOg/xuyQ18PoWwXCzzgpZAiDxL9ktl3XTh8AqoDT5kAqRg+DU48XAGPsUJL2Rn6Bx3Lw==" + }, "packages/tools": { "name": "@web3-storage/tools", "version": "1.0.0", @@ -27048,7 +27054,7 @@ }, "packages/website": { "name": "@web3-storage/website", - "version": "2.37.0", + "version": "2.38.2", "dependencies": { "@docsearch/react": "^3.0.0", "@fortawesome/free-brands-svg-icons": "^6.1.2", diff --git a/packages/api/src/magic.link.js b/packages/api/src/magic.link.js index 9e58150718..af39613404 100644 --- a/packages/api/src/magic.link.js +++ b/packages/api/src/magic.link.js @@ -58,7 +58,12 @@ function createMagicTestmodeBypasss () { export const magicLinkBypassForE2ETestingInTestmode = createMagicTestmodeBypasss() function isMagicTestModeToken (token) { - const parsed = JSON.parse(globalThis.atob(token)) + let parsed + try { + parsed = JSON.parse(globalThis.atob(token)) + } catch { + return false + } if (parsed.length !== 2) { // unexpeced parse return false diff --git a/packages/api/test/fixtures/pgrest/get-user-uploads.js b/packages/api/test/fixtures/pgrest/get-user-uploads.js index 7756732fea..151eb7f1fb 100644 --- a/packages/api/test/fixtures/pgrest/get-user-uploads.js +++ b/packages/api/test/fixtures/pgrest/get-user-uploads.js @@ -29,7 +29,8 @@ export default [ } ], type: 'Car', - updated: '2021-07-14T19:27:14.934572+00:00' + updated: '2021-07-14T19:27:14.934572+00:00', + parts: [] }, { _id: '8', @@ -48,7 +49,8 @@ export default [ } ], type: 'Car', - updated: '2021-07-14T19:27:14.934572+00:00' + updated: '2021-07-14T19:27:14.934572+00:00', + parts: [] }, { _id: '1', @@ -59,7 +61,8 @@ export default [ created: '2021-07-09T16:20:33.946845+00:00', updated: '2021-07-09T16:20:33.946845+00:00', deals: [], - pins: [] + pins: [], + parts: [] }, { _id: '2', @@ -70,7 +73,8 @@ export default [ created: '2021-07-09T10:40:35.408884+00:00', updated: '2021-07-09T10:40:35.408884+00:00', deals: [], - pins: [] + pins: [], + parts: [] }, { _id: '3', @@ -81,6 +85,7 @@ export default [ created: '2021-07-09T10:36:05.862862+00:00', updated: '2021-07-09T10:36:05.862862+00:00', deals: [], - pins: [] + pins: [], + parts: [] } ] diff --git a/packages/db/db-client-types.ts b/packages/db/db-client-types.ts index 61095164e9..d703463375 100644 --- a/packages/db/db-client-types.ts +++ b/packages/db/db-client-types.ts @@ -206,6 +206,7 @@ export type UploadItem = { created?: definitions['upload']['inserted_at'] updated?: definitions['upload']['updated_at'] content: ContentItem + backupUrls: definitions['upload']['backup_urls'] } export type UploadItemOutput = { @@ -218,6 +219,11 @@ export type UploadItemOutput = { dagSize?: definitions['content']['dag_size'] pins: Array, deals: Array + /** + * the graph from `cid` can be recreated from the blocks in these parts + * @see https://github.com/web3-storage/content-claims#partition-claim + */ + parts: Array } export type UploadOutput = definitions['upload'] & { diff --git a/packages/db/index.js b/packages/db/index.js index 21d28b33f4..df5d7d042f 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -21,6 +21,7 @@ const uploadQuery = ` sourceCid:source_cid, created:inserted_at, updated:updated_at, + backupUrls:backup_urls, content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region))) ` @@ -555,7 +556,6 @@ export class DBClient { // Get deals const cids = uploads?.map((u) => u.content.cid) const deals = await this.getDealsForCids(cids) - return { count, uploads: uploads?.map((u) => ({ diff --git a/packages/db/package.json b/packages/db/package.json index 5109f01910..0f9308ffc0 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -22,6 +22,7 @@ "license": "(Apache-2.0 OR MIT)", "dependencies": { "@supabase/postgrest-js": "^0.37.0", + "multiformats": "^13.0.0", "p-retry": "^4.6.1" }, "devDependencies": { diff --git a/packages/db/postgres/functions.sql b/packages/db/postgres/functions.sql index b91f23e80f..e3085b6811 100644 --- a/packages/db/postgres/functions.sql +++ b/packages/db/postgres/functions.sql @@ -10,6 +10,7 @@ DROP FUNCTION IF EXISTS user_used_storage; DROP FUNCTION IF EXISTS user_auth_keys_list; DROP FUNCTION IF EXISTS find_deals_by_content_cids; DROP FUNCTION IF EXISTS upsert_user; +DROP TYPE IF EXISTS stored_bytes; -- transform a JSON array property into an array of SQL text elements CREATE OR REPLACE FUNCTION json_arr_to_text_arr(_json json) diff --git a/packages/db/postgres/reset.sql b/packages/db/postgres/reset.sql index 74b1006e73..badcba45a7 100644 --- a/packages/db/postgres/reset.sql +++ b/packages/db/postgres/reset.sql @@ -11,12 +11,18 @@ DROP TABLE IF EXISTS pin_sync_request; DROP TABLE IF EXISTS psa_pin_request; DROP TABLE IF EXISTS content; DROP TABLE IF EXISTS backup; +DROP TABLE IF EXISTS auth_key_history; DROP TABLE IF EXISTS auth_key; -DROP TABLE IF EXISTS public.user; DROP TABLE IF EXISTS user_tag; DROP TABLE IF EXISTS user_tag_proposal; +DROP TABLE IF EXISTS email_history; +DROP TABLE IF EXISTS user_customer; +DROP TABLE IF EXISTS agreement; +DROP TABLE IF EXISTS public.user; DROP TABLE IF EXISTS terms_of_service; +DROP TYPE IF EXISTS stored_bytes; + DROP SCHEMA IF EXISTS cargo CASCADE; DROP SERVER IF EXISTS dag_cargo_server CASCADE; DROP MATERIALIZED VIEW IF EXISTS public.aggregate_entry CASCADE; diff --git a/packages/db/postgres/tables.sql b/packages/db/postgres/tables.sql index 40d2a78514..c46edf4cbc 100644 --- a/packages/db/postgres/tables.sql +++ b/packages/db/postgres/tables.sql @@ -300,6 +300,7 @@ CREATE INDEX IF NOT EXISTS pin_sync_request_inserted_at_idx ON pin_sync_request -- Setting search_path to public scope for uuid function(s) SET search_path TO public; +DROP TABLE IF EXISTS psa_pin_request; DROP extension IF EXISTS "uuid-ossp"; CREATE extension "uuid-ossp" SCHEMA public; @@ -356,6 +357,8 @@ CREATE TABLE IF NOT EXISTS email_history sent_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL ); + +DROP VIEW IF EXISTS admin_search; CREATE VIEW admin_search as select u.id::text as user_id, diff --git a/packages/db/test/upload.spec.js b/packages/db/test/upload.spec.js index f80c0e0bfc..25f16e8edf 100644 --- a/packages/db/test/upload.spec.js +++ b/packages/db/test/upload.spec.js @@ -278,6 +278,41 @@ describe('upload', () => { assert.ok(userUploads.find(upload => upload.cid === sourceCid)) }) + it('lists user uploads with CAR links in parts', async () => { + const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' + const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR' + const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car' + const exampleS3Url = 'https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/bafybeiao32xtnrlibcekpw3vyfi5txgrmvvrua4pccx3xik33ll3qhko2q/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car' + const created = new Date().toISOString() + const name = `rand-${Math.random().toString().slice(2)}` + await client.createUpload({ + user: user._id, + contentCid, + sourceCid, + authKey: authKeys[0]._id, + type, + dagSize, + name, + pins: [initialPinData], + backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl, exampleS3Url], + created + }) + + // Default sort {inserted_at, Desc} + const { uploads } = await client.listUploads(user._id, { page: 1 }) + assert.ok(uploads.length > 0) + for (const upload of uploads) { + // backupUrls raw is private + assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property') + assert.ok(Array.isArray(upload.parts), 'upload.parts is an array') + } + const namedUpload = uploads.find(u => u.name === name) + assert.deepEqual(namedUpload.parts, [ + // this corresponds to `exampleCarParkUrl` + 'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea' + ]) + }) + it('can list user uploads with several options', async () => { const { uploads: previousUserUploads, count: previousUserUploadCount } = await client.listUploads(user._id, { page: 1 }) assert(previousUserUploads, 'user has uploads') diff --git a/packages/db/utils.js b/packages/db/utils.js index 626a851b80..b040f9d7a6 100644 --- a/packages/db/utils.js +++ b/packages/db/utils.js @@ -1,3 +1,7 @@ +import * as Link from 'multiformats/link' +import * as Digest from 'multiformats/hashes/digest' +import { fromString } from 'uint8arrays' + /** * Normalize upload item. * @@ -6,17 +10,44 @@ */ export function normalizeUpload (upload) { const nUpload = { ...upload } + const backupUrls = nUpload.backupUrls ?? [] + delete nUpload.backupUrls delete nUpload.content delete nUpload.sourceCid + /** @type {import('./db-client-types').UploadItemOutput['parts']} */ + const parts = [...carCidV1Base32sFromBackupUrls(backupUrls)] + return { ...nUpload, ...upload.content, cid: upload.sourceCid, // Overwrite cid to source cid pins: normalizePins(upload.content.pins, { isOkStatuses: true - }) + }), + parts + } +} + +/** + * given array of backup_urls from uploads table, return a corresponding set of CAR CIDv1 using base32 multihash + * for any CAR files in the backup_urls. + * @param {string[]} backupUrls + * @returns {Iterable} + */ +function carCidV1Base32sFromBackupUrls (backupUrls) { + const carCidStrings = new Set() + for (const backupUrl of backupUrls) { + let carCid + try { + carCid = bucketKeyToPartCID(backupUrl) + } catch (error) { + console.warn('error extracting car CID from bucket URL', error) + } + if (!carCid) continue + carCidStrings.add(carCid.toString()) } + return carCidStrings } /** @@ -132,3 +163,30 @@ export function safeNumber (num) { } return num } + +const CAR_CODE = 0x0202 + +/** + * Attempts to extract a CAR CID from a bucket key. + * + * @param {string} key + */ +const bucketKeyToPartCID = key => { + const filename = String(key.split('/').at(-1)) + const [hash] = filename.split('.') + try { + // recent buckets encode CAR CID in filename + const cid = Link.parse(hash).toV1() + if (cid.code === CAR_CODE) return cid + throw new Error('not a CAR CID') + } catch (err) { + // older buckets base32 encode a CAR multihash .car + try { + const digestBytes = fromString(hash, 'base32') + const digest = Digest.decode(digestBytes) + return Link.create(CAR_CODE, digest) + } catch (error) { + // console.warn('error trying to create CID from s3 key', error) + } + } +}