feat(ipos): car file support

services/ipos/biome.json

@@ -0,0 +1,21 @@
+{
+	"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
+	"organizeImports": {
+		"enabled": true
+	},
+	"files": {
+		"ignore": ["node_modules", ".wrangler", ".papi"]
+	},
+	"linter": {
+		"enabled": true,
+		"rules": {
+			"recommended": true
+		}
+	},
+	"javascript": {
+		"formatter": {
+			"semicolons": "asNeeded",
+			"quoteStyle": "single"
+		}
+	}
+}

services/ipos/package.json

@@ -1,31 +1,31 @@
 {
-  "name": "ipos",
-  "version": "0.0.0",
-  "private": true,
-  "scripts": {
-    "deploy": "wrangler deploy",
-    "dev": "wrangler dev",
-    "start": "wrangler dev",
-    "test": "vitest",
-    "cf-typegen": "wrangler types"
-  },
-  "devDependencies": {
-    "@cloudflare/vitest-pool-workers": "^0.4.5",
-    "@cloudflare/workers-types": "^4.20240620.0",
-    "typescript": "^5.4.5",
-    "vitest": "1.5.0",
-    "wrangler": "^3.60.3"
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.609.0",
-    "@helia/unixfs": "^3.0.6",
-    "@hono/valibot-validator": "^0.2.5",
-    "blockstore-core": "^4.4.1",
-    "helia": "^4.2.4",
-    "hono": "^4.4.12",
-    "ipfs-only-hash": "^4.0.0",
-    "ipfs-unixfs-importer": "^15.2.5",
-    "multiformats": "^13.1.3",
-    "valibot": "^0.30.0"
-  }
+	"name": "ipos",
+	"version": "0.0.0",
+	"private": true,
+	"scripts": {
+		"deploy": "wrangler deploy",
+		"dev": "wrangler dev",
+		"start": "wrangler dev",
+		"test": "vitest",
+		"cf-typegen": "wrangler types",
+		"lint": "biome check --write ."
+	},
+	"devDependencies": {
+		"@biomejs/biome": "^1.9.2",
+		"@cloudflare/vitest-pool-workers": "^0.4.5",
+		"@cloudflare/workers-types": "^4.20240620.0",
+		"typescript": "^5.4.5",
+		"vitest": "1.5.0",
+		"wrangler": "^3.60.3"
+	},
+	"dependencies": {
+		"@aws-sdk/client-s3": "^3.609.0",
+		"@hono/valibot-validator": "^0.2.5",
+		"hono": "^4.4.12",
+		"ipfs-car": "0.9.2",
+		"ipfs-only-hash": "^4.0.0",
+		"ipfs-unixfs-importer": "^15.2.5",
+		"multiformats": "^13.1.3",
+		"valibot": "^0.30.0"
+	}
 }

services/ipos/src/index.ts

@@ -1,19 +1,19 @@
-import { Hono } from 'hono';
-import { cors } from 'hono/cors';
+import { Hono } from 'hono'
+import { cors } from 'hono/cors'
 
-import { HonoEnv } from './utils/constants';
-import { pinning } from './routes/pinning';
+import { pinning } from './routes/pinning'
+import type { HonoEnv } from './utils/constants'
 
-const app = new Hono<HonoEnv>();
+const app = new Hono<HonoEnv>()
 
-app.get('/', (c) => c.text('Hello <<Artists>>!'));
-app.use('*', cors());
+app.get('/', (c) => c.text('Hello <<Artists>>!'))
+app.use('*', cors())
 
-app.route('/', pinning);
+app.route('/', pinning)
 
 app.onError((err, c) => {
-	console.error(`${err}`);
-	return c.json({ error: err.message, path: c.req.url }, 400);
-});
+	console.error(`${err}`)
+	return c.json({ error: err.message, path: c.req.url }, 400)
+})
 
-export default app;
+export default app

services/ipos/src/routes/pinning.ts

@@ -1,37 +1,44 @@
-import { Hono } from 'hono'
-import { HonoEnv } from '../utils/constants'
 import { vValidator } from '@hono/valibot-validator'
-import { blob, object, union, array } from 'valibot'
-import { getUint8ArrayFromFile, getObjectSize, hashOf, keyOf } from '../utils/format'
+import { Hono } from 'hono'
+import { array, blob, object, union } from 'valibot'
+import type { HonoEnv } from '../utils/constants'
+import {
+	getObjectSize,
+	getUint8ArrayFromFile,
+	hashOf,
+	keyOf,
+} from '../utils/format'
+import toCar from '../utils/ipfs'
 import { getS3 } from '../utils/s3'
-import { getDirectoryCID } from '../utils/helia'
 
 const app = new Hono<HonoEnv>()
 
 app.post('/pinJson', vValidator('json', object({})), async (c) => {
-  const body = await c.req.json()
-  const type = 'application/json'
-  const s3 = getS3(c)
-
-  const content = JSON.stringify(body)
-  const cid = (await hashOf(content)).toV0().toString()
-
-  await s3.putObject({
-    Body: content,
-    Bucket: c.env.FILEBASE_BUCKET_NAME,
-    Key: cid,
-    ContentType: type,
-  })
-
-  c.executionCtx.waitUntil(c.env.BUCKET.put(keyOf(cid), new Blob([content], { type })))
-
-  return c.json(
-    getPinResponse({
-      cid: cid,
-      type: type,
-      size: getObjectSize(body),
-    }),
-  )
+	const body = await c.req.json()
+	const type = 'application/json'
+	const s3 = getS3(c)
+
+	const content = JSON.stringify(body)
+	const cid = (await hashOf(content)).toV0().toString()
+
+	await s3.putObject({
+		Body: content,
+		Bucket: c.env.FILEBASE_BUCKET_NAME,
+		Key: cid,
+		ContentType: type,
+	})
+
+	c.executionCtx.waitUntil(
+		c.env.BUCKET.put(keyOf(cid), new Blob([content], { type })),
+	)
+
+	return c.json(
+		getPinResponse({
+			cid: cid,
+			type: type,
+			size: getObjectSize(body),
+		}),
+	)
 })
 
 const fileRequiredMessage = 'File is required'
@@ -40,85 +47,82 @@ const fileKey = 'file'
 type PinFile = { [fileKey]: File } | { [fileKey]: File[] }
 
 const pinFileRequestSchema = object({
-  [fileKey]: union([
-    blob(fileRequiredMessage),
-    array(blob(fileRequiredMessage)),
-  ]),
+	[fileKey]: union([
+		blob(fileRequiredMessage),
+		array(blob(fileRequiredMessage)),
+	]),
 })
 
 app.post('/pinFile', vValidator('form', pinFileRequestSchema), async (c) => {
-  const body = (await c.req.parseBody({ all: true })) as PinFile
-
-  const files = await Promise.all(
-    ([[body[fileKey]]].flat(2).filter(Boolean) as File[]).map(async (file) => ({
-      file,
-      content: await getUint8ArrayFromFile(file),
-    })),
-  )
-
-  const hasMultipleFiles = files.length > 1
-  const s3 = getS3(c)
-
-  let directoryCId: string | undefined
-
-  if (hasMultipleFiles) {
-    directoryCId = (await getDirectoryCID({ files, c })).toV0().toString()
-  }
-
-  const addedFiles: { file: File; cid: string; content: Uint8Array }[] =
-    await Promise.all(
-      files.map(async ({ file, content }) => {
-        try {
-          const cid = (await hashOf(content)).toV0().toString()
-          const prefix = directoryCId ? `${directoryCId}/` : ''
-
-          await s3.putObject({
-            Body: content,
-            Bucket: c.env.FILEBASE_BUCKET_NAME,
-            Key: `${prefix}${cid}`,
-            ContentType: file.type,
-          })
-
-          console.log('File added', cid)
-          return { file, cid, content }
-        } catch (error) {
-          throw new Error(`Failed to add file ${file.name}: ${error?.message}`)
-        }
-      }),
-    )
-
-  c.executionCtx.waitUntil(
-    Promise.all(
-      addedFiles.map(({ content, cid }) => c.env.BUCKET.put(keyOf(cid), content)),
-    ),
-  )
-
-  const size = files.reduce((reducer, file) => reducer + file.file.size, 0)
-  const { cid: addedFileCid, file } = addedFiles[0]
-  let cid = addedFileCid
-  let type = file.type
-
-  if (hasMultipleFiles) {
-    cid = directoryCId as string
-    type = 'directory'
-  }
-
-  return c.json(
-    getPinResponse({
-      cid: cid,
-      type: type,
-      size: size,
-    }),
-  )
+	const body = (await c.req.parseBody({ all: true })) as PinFile
+
+	const files = await Promise.all(
+		([[body[fileKey]]].flat(2).filter(Boolean) as File[]).map(async (file) => ({
+			file,
+			content: await getUint8ArrayFromFile(file),
+		})),
+	)
+
+	const s3 = getS3(c)
+
+	let cid: string
+	let file: Blob | File
+	let type: string
+
+	if (files.length > 1) {
+		const { root, car } = await toCar(
+			files.map(({ file, content }) => ({
+				path: file.name,
+				content: content,
+			})),
+		)
+
+		cid = root.toString()
+		file = car
+		type = 'directory'
+
+		await s3.putObject({
+			Body: car,
+			Bucket: c.env.FILEBASE_BUCKET_NAME,
+			Key: cid,
+			ContentType: 'application/vnd.ipld.car',
+			Metadata: {
+				import: 'car',
+			},
+		})
+	} else {
+		const { content, file: f } = files[0]
+		cid = (await hashOf(content)).toV0().toString()
+
+		file = f
+		type = f.type
+
+		await s3.putObject({
+			Body: content,
+			Bucket: c.env.FILEBASE_BUCKET_NAME,
+			Key: cid,
+			ContentType: f.type,
+		})
+	}
+
+	c.executionCtx.waitUntil(c.env.BUCKET.put(keyOf(cid), file))
+
+	return c.json(
+		getPinResponse({
+			cid: cid,
+			type: type,
+			size: file.size,
+		}),
+	)
 })
 
 const getPinResponse = (value: {
-  cid: string
-  type: string
-  size: number
+	cid: string
+	type: string
+	size: number
 }) => ({
-  ok: true,
-  value,
+	ok: true,
+	value,
 })
 
 export { app as pinning }

services/ipos/src/utils/constants.ts

@@ -1,19 +1,19 @@
-import { Env } from 'hono'
+import type { Env } from 'hono'
 
-export interface CloudflareEnv extends Record<string, any> {
-  BUCKET: R2Bucket
+export interface CloudflareEnv extends Record<string, unknown> {
+	BUCKET: R2Bucket
 
-  // wrangler secret
-  // S3
-  S3_ACCESS_KEY_ID: string
-  S3_SECRET_ACCESS_KEY: string
-  
-  // Filebase
-  FILEBASE_BUCKET_NAME: string
+	// wrangler secret
+	// S3
+	S3_ACCESS_KEY_ID: string
+	S3_SECRET_ACCESS_KEY: string
+
+	// Filebase
+	FILEBASE_BUCKET_NAME: string
 }
 
 export interface HonoEnv extends Env {
-  Bindings: CloudflareEnv
+	Bindings: CloudflareEnv
 }
 
 export const ORIGIN = 'https://kodadot.xyz'