feat: listUploads includes parts property with list of links to CARs that contain it

packages/api/src/magic.link.js

@@ -58,7 +58,12 @@ function createMagicTestmodeBypasss () {
 export const magicLinkBypassForE2ETestingInTestmode = createMagicTestmodeBypasss()
 
 function isMagicTestModeToken (token) {
-  const parsed = JSON.parse(globalThis.atob(token))
+  let parsed
+  try {
+    parsed = JSON.parse(globalThis.atob(token))
+  } catch (error) {
+    return false
+  }
   if (parsed.length !== 2) {
     // unexpeced parse
     return false

packages/db/db-client-types.ts

@@ -206,6 +206,7 @@ export type UploadItem = {
   created?: definitions['upload']['inserted_at']
   updated?: definitions['upload']['updated_at']
   content: ContentItem
+  backupUrls: definitions['upload']['backup_urls']
 }
 
 export type UploadItemOutput = {
@@ -218,6 +219,8 @@ export type UploadItemOutput = {
   dagSize?: definitions['content']['dag_size']
   pins: Array<PinItemOutput>,
   deals: Array<Deal>
+  // array of links to things containing this Upload (e.g. CARs)
+  partOf: Array<string>
 }
 
 export type UploadOutput = definitions['upload'] & {

packages/db/index.js

@@ -21,6 +21,7 @@ const uploadQuery = `
         sourceCid:source_cid,
         created:inserted_at,
         updated:updated_at,
+        backupUrls:backup_urls,
         content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region)))
       `
 
@@ -555,7 +556,6 @@ export class DBClient {
     // Get deals
     const cids = uploads?.map((u) => u.content.cid)
     const deals = await this.getDealsForCids(cids)
-
     return {
       count,
       uploads: uploads?.map((u) => ({

packages/db/postgres/functions.sql

@@ -10,6 +10,7 @@ DROP FUNCTION IF EXISTS user_used_storage;
 DROP FUNCTION IF EXISTS user_auth_keys_list;
 DROP FUNCTION IF EXISTS find_deals_by_content_cids;
 DROP FUNCTION IF EXISTS upsert_user;
+DROP TYPE IF EXISTS stored_bytes;
 
 -- transform a JSON array property into an array of SQL text elements
 CREATE OR REPLACE FUNCTION json_arr_to_text_arr(_json json)

packages/db/postgres/reset.sql

@@ -11,12 +11,18 @@ DROP TABLE IF EXISTS pin_sync_request;
 DROP TABLE IF EXISTS psa_pin_request;
 DROP TABLE IF EXISTS content;
 DROP TABLE IF EXISTS backup;
+DROP TABLE IF EXISTS auth_key_history;
 DROP TABLE IF EXISTS auth_key;
-DROP TABLE IF EXISTS public.user;
 DROP TABLE IF EXISTS user_tag;
 DROP TABLE IF EXISTS user_tag_proposal;
+DROP TABLE IF EXISTS email_history;
+DROP TABLE IF EXISTS user_customer;
+DROP TABLE IF EXISTS agreement;
+DROP TABLE IF EXISTS public.user;
 DROP TABLE IF EXISTS terms_of_service;
 
+DROP TYPE IF EXISTS stored_bytes;
+
 DROP SCHEMA IF EXISTS cargo CASCADE;
 DROP SERVER IF EXISTS dag_cargo_server CASCADE;
 DROP MATERIALIZED VIEW IF EXISTS public.aggregate_entry CASCADE;

packages/db/postgres/tables.sql

@@ -300,6 +300,7 @@ CREATE INDEX IF NOT EXISTS pin_sync_request_inserted_at_idx ON pin_sync_request
 
 -- Setting search_path to public scope for uuid function(s)
 SET search_path TO public;
+DROP TABLE IF EXISTS psa_pin_request;
 DROP extension IF EXISTS "uuid-ossp";
 CREATE extension "uuid-ossp" SCHEMA public;
 
@@ -356,6 +357,8 @@ CREATE TABLE IF NOT EXISTS email_history
   sent_at         TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL
 );
 
+
+DROP VIEW IF EXISTS admin_search;
 CREATE VIEW admin_search as
 select
   u.id::text as user_id,

packages/db/test/upload.spec.js

@@ -278,6 +278,40 @@ describe('upload', () => {
     assert.ok(userUploads.find(upload => upload.cid === sourceCid))
   })
 
+  it('lists user uploads with CAR links in partOf', async () => {
+    const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi'
+    const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR'
+    const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car'
+    const created = new Date().toISOString()
+    const name = `rand-${Math.random().toString().slice(2)}`
+    await client.createUpload({
+      user: user._id,
+      contentCid,
+      sourceCid,
+      authKey: authKeys[0]._id,
+      type,
+      dagSize,
+      name,
+      pins: [initialPinData],
+      backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl],
+      created
+    })
+
+    // Default sort {inserted_at, Desc}
+    const { uploads } = await client.listUploads(user._id, { page: 1 })
+    assert.ok(uploads.length > 0)
+    for (const upload of uploads) {
+      // backupUrls raw is private
+      assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property')
+      assert.ok(Array.isArray(upload.partOf), 'upload.partOf is an array')
+    }
+    const namedUpload = uploads.find(u => u.name === name)
+    assert.deepEqual(namedUpload.partOf, [
+      // this corresponds to `exampleCarParkUrl`
+      'ipfs://bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea'
+    ])
+  })
+
   it('can list user uploads with several options', async () => {
     const { uploads: previousUserUploads, count: previousUserUploadCount } = await client.listUploads(user._id, { page: 1 })
     assert(previousUserUploads, 'user has uploads')

packages/db/utils.js

@@ -6,17 +6,39 @@
  */
 export function normalizeUpload (upload) {
   const nUpload = { ...upload }
+  const backupUrls = nUpload.backupUrls ?? []
+  delete nUpload.backupUrls
   delete nUpload.content
   delete nUpload.sourceCid
 
+  const partOf = [...carUrlsFromBackupUrls(backupUrls)]
+
   return {
     ...nUpload,
     ...upload.content,
     cid: upload.sourceCid, // Overwrite cid to source cid
     pins: normalizePins(upload.content.pins, {
       isOkStatuses: true
-    })
+    }),
+    partOf
+  }
+}
+
+/**
+ * given array of backup_urls from uploads table, return a set of ipfs:// URIs for any CAR files in the backup_urls
+ * @param {string[]} backupUrls
+ * @returns {Iterable<string>}
+ */
+function carUrlsFromBackupUrls (backupUrls) {
+  const carCIDUrls = new Set()
+  for (const backupUrl of backupUrls) {
+    // match cid v1 starting with 'ba'.
+    // there are also backupUrls from s3 with .car suffix and path stem is base32(multihash) (not a CID). exclude those.
+    const carCidFileSuffixMatch = String(backupUrl).match(/\/(ba[^/]+).car$/)
+    if (!carCidFileSuffixMatch) continue
+    carCIDUrls.add(`ipfs://${carCidFileSuffixMatch[1]}`)
   }
+  return carCIDUrls
 }
 
 /**