fix: issues with aws s3 integration (#50)

* fix: issues with aws s3 integration

* fix: ensure bucket exist for cache initialization

app/api/report/generate/route.ts

@@ -15,16 +15,21 @@ export async function POST(request: Request) {
   const { result: reportId, error } = await withError(service.generateReport(resultsIds, { project, ...rest }));
 
   if (error) {
+    console.error(error);
+
     return new Response(error.message, { status: 404 });
   }
 
   if (!reportId) {
     return new Response('failed to generate report', { status: 400 });
   }
 
+  const projectPath = project ? `${encodeURI(project)}/` : '';
+  const reportUrl = `${serveReportRoute}/${projectPath}${reportId}/index.html`;
+
   return Response.json({
     reportId,
     project,
-    reportUrl: `${serveReportRoute}/${project ? encodeURI(project) : ''}/${reportId}/index.html`,
+    reportUrl,
   });
 }

app/lib/pw.ts

@@ -5,6 +5,7 @@ import path from 'node:path';
 
 import { withError } from './withError';
 import { REPORTS_FOLDER, TMP_FOLDER } from './storage/constants';
+import { createDirectory } from './storage/folders';
 
 const execAsync = util.promisify(exec);
 
@@ -16,6 +17,8 @@ export const generatePlaywrightReport = async (
 
   const reportPath = path.join(REPORTS_FOLDER, projectName ?? '', reportId);
 
+  await createDirectory(reportPath);
+
   console.log(`[pw] report path: ${reportPath}`);
 
   const tempFolder = path.join(TMP_FOLDER, reportId);

app/lib/storage/folders.ts

@@ -0,0 +1,10 @@
+import fs from 'node:fs/promises';
+
+export async function createDirectory(dir: string) {
+  try {
+    await fs.access(dir);
+  } catch {
+    await fs.mkdir(dir, { recursive: true });
+    console.log('Created directory:', dir);
+  }
+}

app/lib/storage/fs.ts

@@ -17,7 +17,8 @@ import {
 } from './constants';
 import { processBatch } from './batch';
 import { handlePagination } from './pagination';
-import { defaultStreamingOptions, transformStreamToReadable } from './stream';
+import { defaultStreamingOptions, transformBlobToReadable } from './stream';
+import { createDirectory } from './folders';
 
 import { parse } from '@/app/lib/parser';
 import { generatePlaywrightReport } from '@/app/lib/pw';
@@ -36,15 +37,6 @@ import {
 } from '@/app/lib/storage';
 
 async function createDirectoriesIfMissing() {
-  async function createDirectory(dir: string) {
-    try {
-      await fs.access(dir);
-    } catch {
-      await fs.mkdir(dir, { recursive: true });
-      console.log('Created directory:', dir);
-    }
-  }
-
   await createDirectory(RESULTS_FOLDER);
   await createDirectory(REPORTS_FOLDER);
   await createDirectory(TMP_FOLDER);
@@ -265,7 +257,7 @@ export async function saveResult(file: Blob, size: number, resultDetails: Result
   const resultID = randomUUID();
   const resultPath = path.join(RESULTS_FOLDER, `${resultID}.zip`);
 
-  const readable = transformStreamToReadable(file.stream());
+  const readable = transformBlobToReadable(file);
   const writeable = createWriteStream(resultPath, defaultStreamingOptions);
 
   /**

app/lib/storage/s3.ts

@@ -27,7 +27,7 @@ import {
 } from './constants';
 import { handlePagination } from './pagination';
 import { getFileReportID } from './file';
-import { transformStreamToReadable } from './stream';
+import { transformBlobToReadable } from './stream';
 
 import { parse } from '@/app/lib/parser';
 import { serveReportRoute } from '@/app/lib/constants';
@@ -110,15 +110,15 @@ export class S3 implements Storage {
   private async write(dir: string, files: { name: string; content: Readable | Buffer | string; size?: number }[]) {
     await this.ensureBucketExist();
     for (const file of files) {
-      const path = `${dir}/${file.name}`;
+      const filePath = path.join(dir, file.name);
 
-      console.log(`[s3] writing ${path}`);
+      console.log(`[s3] writing ${filePath}`);
 
       const content = typeof file.content === 'string' ? Buffer.from(file.content) : file.content;
 
       const contentSize = file.size ?? (Buffer.isBuffer(content) ? content.length : undefined);
 
-      await this.client.putObject(this.bucket, path, content, contentSize);
+      await this.client.putObject(this.bucket, path.normalize(filePath), content, contentSize);
     }
   }
 
@@ -182,7 +182,7 @@ export class S3 implements Storage {
           indexCount += 1;
         }
 
-        totalSize += obj.size;
+        totalSize += obj?.size ?? 0;
       });
 
       stream.on('error', (err) => {
@@ -228,6 +228,8 @@ export class S3 implements Storage {
   }
 
   async readResults(input?: ReadResultsInput): Promise<ReadResultsOutput> {
+    await this.ensureBucketExist();
+
     console.log('[s3] reading results');
     const listResultsStream = this.client.listObjectsV2(this.bucket, RESULTS_BUCKET, true);
 
@@ -264,7 +266,7 @@ export class S3 implements Storage {
 
     const { result: jsonFiles } = await withError(findJsonFiles);
 
-    console.log(`[s3] found ${jsonFiles?.length} json files`);
+    console.log(`[s3] found ${(jsonFiles ?? [])?.length} json files`);
 
     if (!jsonFiles) {
       return {
@@ -316,7 +318,9 @@ export class S3 implements Storage {
   }
 
   async readReports(input?: ReadReportsInput): Promise<ReadReportsOutput> {
-    console.log(`[s3] reading reports from minio`);
+    await this.ensureBucketExist();
+
+    console.log(`[s3] reading reports from external storage`);
     const reportsStream = this.client.listObjectsV2(this.bucket, REPORTS_BUCKET, true);
 
     const reports: Report[] = [];
@@ -517,7 +521,7 @@ export class S3 implements Storage {
     await this.write(RESULTS_BUCKET, [
       {
         name: `${resultID}.zip`,
-        content: transformStreamToReadable(file.stream()),
+        content: transformBlobToReadable(file),
         size,
       },
       {
@@ -542,7 +546,7 @@ export class S3 implements Storage {
       console.log(`[s3] uploading file: ${JSON.stringify(file)}`);
 
       const nestedPath = file.path.split(reportId).pop();
-      const s3Path = `/${remotePath}/${nestedPath}/${file.name}`;
+      const s3Path = path.join(remotePath, nestedPath ?? '', file.name);
 
       console.log(`[s3] uploading to ${s3Path}`);
 
@@ -559,7 +563,7 @@ export class S3 implements Storage {
     if (attempt > 3) {
       throw new Error(`[s3] failed to upload file after ${attempt} attempts: ${filePath}`);
     }
-    const { error } = await withError(this.client.fPutObject(this.bucket, remotePath, filePath));
+    const { error } = await withError(this.client.fPutObject(this.bucket, remotePath, filePath, {}));
 
     if (error) {
       console.error(`[s3] failed to upload file: ${error.message}`);
@@ -611,6 +615,8 @@ export class S3 implements Storage {
         const { error } = await withError(this.client.fGetObject(this.bucket, result.name, localFilePath));
 
         if (error) {
+          console.error(`[s3] failed to download ${result.name}: ${error.message}`);
+
           throw new Error(`failed to download ${result.name}: ${error.message}`);
         }
 

app/lib/storage/stream.ts

@@ -1,4 +1,5 @@
 import { Readable, type ReadableOptions } from 'node:stream';
+import { ReadableStream } from 'node:stream/web';
 
 /**
  * convert a formData file multipart stream to a readable node stream
@@ -7,24 +8,8 @@ import { Readable, type ReadableOptions } from 'node:stream';
  * Web stream: https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream
  * Node stream: https://nodejs.org/docs/latest-v20.x/api/stream.html#readable-streams
  */
-export const transformStreamToReadable = (stream: ReadableStream<Uint8Array>, opts?: ReadableOptions): Readable => {
-  return Readable.from(
-    (async function* () {
-      const reader = stream.getReader();
-
-      try {
-        while (true) {
-          const { done, value } = await reader.read();
-
-          if (done) break;
-          yield value;
-        }
-      } finally {
-        reader.releaseLock();
-      }
-    })(),
-    opts ?? defaultStreamingOptions,
-  );
+export const transformBlobToReadable = (blob: Blob, opts?: ReadableOptions): Readable => {
+  return Readable.fromWeb(blob.stream() as ReadableStream<never>, opts);
 };
 
 export const defaultStreamingOptions: ReadableOptions = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "playwright-reports-server",
-  "version": "4.2.0",
+  "version": "4.2.1",
   "description": "Playwright Reports Server: Can be used to accept blobs, merge them, store and view Playwright reports",
   "scripts": {
     "build": "next build",