From b2e80e1d53c4e36e0fd073a40c4033a192ed54e4 Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 4 Oct 2024 19:58:43 +0200 Subject: [PATCH 01/32] Added multi-file write support --- .../js-sdk/src/sandbox/filesystem/index.ts | 149 ++++++++++-------- .../js-sdk/tests/sandbox/files/write.test.ts | 56 ++++++- 2 files changed, 138 insertions(+), 67 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index a9f3ef838..490ce91ba 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -1,18 +1,5 @@ -import { - createPromiseClient, - Transport, - PromiseClient, - ConnectError, - Code, -} from '@connectrpc/connect' -import { - ConnectionConfig, - defaultUsername, - Username, - ConnectionOpts, - KEEPALIVE_PING_INTERVAL_SEC, - KEEPALIVE_PING_HEADER, -} from '../../connectionConfig' +import { Code, ConnectError, createPromiseClient, PromiseClient, Transport } from '@connectrpc/connect' +import { ConnectionConfig, ConnectionOpts, defaultUsername, KEEPALIVE_PING_HEADER, KEEPALIVE_PING_INTERVAL_SEC, Username } from '../../connectionConfig' import { handleEnvdApiError, handleWatchDirStartEvent } from '../../envd/api' import { authenticationHeader, handleRpcError } from '../../envd/rpc' @@ -21,7 +8,7 @@ import { EnvdApiClient } from '../../envd/api' import { Filesystem as FilesystemService } from '../../envd/filesystem/filesystem_connect' import { FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' -import { WatchHandle, FilesystemEvent } from './watchHandle' +import { FilesystemEvent, WatchHandle } from './watchHandle' export interface EntryInfo { name: string @@ -34,6 +21,8 @@ export const enum FileType { DIR = 'dir', } +export type WriteData = string | ArrayBuffer | Blob | ReadableStream + function mapFileType(fileType: FsFileType) { switch (fileType) { case FsFileType.DIRECTORY: @@ -57,11 +46,7 @@ export class Filesystem { private readonly defaultWatchTimeout = 60_000 // 60 seconds - constructor( - transport: Transport, - private readonly envdApi: EnvdApiClient, - private readonly connectionConfig: ConnectionConfig, - ) { + constructor(transport: Transport, private readonly envdApi: EnvdApiClient, private readonly connectionConfig: ConnectionConfig) { this.rpc = createPromiseClient(FilesystemService, transport) } @@ -95,8 +80,17 @@ export class Filesystem { return res.data } - async write(path: string, data: string | ArrayBuffer | Blob | ReadableStream, opts?: FilesystemRequestOpts): Promise { - const blob = await new Response(data).blob() + async write(path: string, data: WriteData | { data: WriteData; filename: string }[], opts?: FilesystemRequestOpts): Promise { + let blobs: Blob[] = [] + if (Array.isArray(data)) { + for (const d of data) { + const blob = await new Response(d.data).blob() + blobs.push(blob) + } + } else { + const blob = await new Response(data).blob() + blobs.push(blob) + } const res = await this.envdApi.api.POST('/files', { params: { @@ -108,7 +102,15 @@ export class Filesystem { bodySerializer() { const fd = new FormData() - fd.append('file', blob) + for (let i = 0; i < blobs.length; i++) { + const blob = blobs[i] + if (Array.isArray(data)) { + const filename = data[i].filename + fd.append('file', blob, filename) + } else { + fd.append('file', blob) + } + } return fd }, @@ -126,15 +128,22 @@ export class Filesystem { throw new Error('Expected to receive information about written file') } - return files[0] as EntryInfo + if (files.length > 1) { + return files as EntryInfo[] + } else { + return files[0] as EntryInfo + } } async list(path: string, opts?: FilesystemRequestOpts): Promise { try { - const res = await this.rpc.listDir({ path }, { - headers: authenticationHeader(opts?.user), - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - }) + const res = await this.rpc.listDir( + { path }, + { + headers: authenticationHeader(opts?.user), + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + } + ) const entries: EntryInfo[] = [] @@ -158,10 +167,13 @@ export class Filesystem { async makeDir(path: string, opts?: FilesystemRequestOpts): Promise { try { - await this.rpc.makeDir({ path }, { - headers: authenticationHeader(opts?.user), - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - }) + await this.rpc.makeDir( + { path }, + { + headers: authenticationHeader(opts?.user), + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + } + ) return true } catch (err) { @@ -177,13 +189,16 @@ export class Filesystem { async rename(oldPath: string, newPath: string, opts?: FilesystemRequestOpts): Promise { try { - const res = await this.rpc.move({ - source: oldPath, - destination: newPath, - }, { - headers: authenticationHeader(opts?.user), - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - }) + const res = await this.rpc.move( + { + source: oldPath, + destination: newPath, + }, + { + headers: authenticationHeader(opts?.user), + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + } + ) const entry = res.entry if (!entry) { @@ -202,10 +217,13 @@ export class Filesystem { async remove(path: string, opts?: FilesystemRequestOpts): Promise { try { - await this.rpc.remove({ path }, { - headers: authenticationHeader(opts?.user), - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - }) + await this.rpc.remove( + { path }, + { + headers: authenticationHeader(opts?.user), + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + } + ) } catch (err) { throw handleRpcError(err) } @@ -213,10 +231,13 @@ export class Filesystem { async exists(path: string, opts?: FilesystemRequestOpts): Promise { try { - await this.rpc.stat({ path }, { - headers: authenticationHeader(opts?.user), - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), - }) + await this.rpc.stat( + { path }, + { + headers: authenticationHeader(opts?.user), + signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + } + ) return true } catch (err) { @@ -233,7 +254,7 @@ export class Filesystem { async watch( path: string, onEvent: (event: FilesystemEvent) => void | Promise, - opts?: FilesystemRequestOpts & { timeout?: number, onExit?: (err?: Error) => void | Promise }, + opts?: FilesystemRequestOpts & { timeout?: number; onExit?: (err?: Error) => void | Promise } ): Promise { const requestTimeoutMs = opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs @@ -241,30 +262,28 @@ export class Filesystem { const reqTimeout = requestTimeoutMs ? setTimeout(() => { - controller.abort() - }, requestTimeoutMs) + controller.abort() + }, requestTimeoutMs) : undefined - const events = this.rpc.watchDir({ path }, { - headers: { - ...authenticationHeader(opts?.user), - [KEEPALIVE_PING_HEADER]: KEEPALIVE_PING_INTERVAL_SEC.toString(), - }, - signal: controller.signal, - timeoutMs: opts?.timeout ?? this.defaultWatchTimeout, - }) + const events = this.rpc.watchDir( + { path }, + { + headers: { + ...authenticationHeader(opts?.user), + [KEEPALIVE_PING_HEADER]: KEEPALIVE_PING_INTERVAL_SEC.toString(), + }, + signal: controller.signal, + timeoutMs: opts?.timeout ?? this.defaultWatchTimeout, + } + ) try { await handleWatchDirStartEvent(events) clearTimeout(reqTimeout) - return new WatchHandle( - () => controller.abort(), - events, - onEvent, - opts?.onExit, - ) + return new WatchHandle(() => controller.abort(), events, onEvent, opts?.onExit) } catch (err) { throw handleRpcError(err) } diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 72d101d41..2e2649ac4 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -1,12 +1,64 @@ -import { assert } from 'vitest' +import { assert, onTestFinished } from 'vitest' +import { EntryInfo } from '../../../src/index.js' +import { WriteData } from '../../../src/sandbox/filesystem/index.js' import { sandboxTest } from '../../setup.js' sandboxTest('write file', async ({ sandbox }) => { const filename = 'test_write.txt' const content = 'This is a test file.' - const info = await sandbox.files.write(filename, content) + const info = (await sandbox.files.write(filename, content)) as EntryInfo + assert.isFalse(Array.isArray(info)) + assert.equal(info.name, filename) + assert.equal(info.type, 'file') + assert.equal(info.path, `/home/user/${filename}`) + + const exists = await sandbox.files.exists(filename) + assert.isTrue(exists) + const readContent = await sandbox.files.read(filename) + assert.equal(readContent, content) +}) + +sandboxTest('write multiple files', async ({ sandbox }) => { + let files: Array<{ data: WriteData; filename: string }> = [] + + for (let i = 0; i < 10; i++) { + const filename = `multi_test_file${i}.txt` + onTestFinished(async () => await sandbox.files.remove(filename)) + + files.push({ + filename: `multi_test_file${i}.txt`, + data: `This is a test file ${i}.`, + }) + } + + const infos = await sandbox.files.write('', files) + + assert.isTrue(Array.isArray(infos)) + assert.equal((infos as EntryInfo[]).length, files.length) + + for (let i = 0; i < files.length; i++) { + const file = files[i] + const info = infos[i] as EntryInfo + + assert.equal(info.name, file.filename) + assert.equal(info.type, 'file') + assert.equal(info.path, `/home/user/${file.filename}`) + + const exists = await sandbox.files.exists(file.filename) + assert.isTrue(exists) + const readContent = await sandbox.files.read(file.filename) + assert.equal(readContent, file.data) + } +}) + +sandboxTest('write file', async ({ sandbox }) => { + const filename = 'test_write.txt' + const content = 'This is a test file.' + + const info = (await sandbox.files.write(filename, content)) as EntryInfo + assert.isFalse(Array.isArray(info)) assert.equal(info.name, filename) assert.equal(info.type, 'file') assert.equal(info.path, `/home/user/${filename}`) From 9e6bc1739cfe5ba7fe72902281f5ae5e12c8b682 Mon Sep 17 00:00:00 2001 From: 0div Date: Fri, 4 Oct 2024 23:13:12 +0200 Subject: [PATCH 02/32] address PR comments --- packages/js-sdk/src/index.ts | 49 ++++--------------- .../js-sdk/src/sandbox/filesystem/index.ts | 24 ++++++--- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index 0425b7097..fdc292238 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -1,55 +1,26 @@ -export { - ApiClient, -} from './api' +export { ApiClient } from './api' export type { components, paths } from './api' -export { - AuthenticationError, - SandboxError, - TimeoutError, - NotFoundError, - NotEnoughSpaceError, - InvalidArgumentError, - TemplateError, -} from './errors' -export { - ConnectionConfig, -} from './connectionConfig' -export type { Logger } from './logs' +export { ConnectionConfig } from './connectionConfig' export type { ConnectionOpts, Username } from './connectionConfig' +export { AuthenticationError, InvalidArgumentError, NotEnoughSpaceError, NotFoundError, SandboxError, TemplateError, TimeoutError } from './errors' +export type { Logger } from './logs' +export { FileType } from './sandbox/filesystem' +export type { EntryInfo, Filesystem, WriteData } from './sandbox/filesystem' export { FilesystemEventType } from './sandbox/filesystem/watchHandle' export type { FilesystemEvent, WatchHandle } from './sandbox/filesystem/watchHandle' -export type { - EntryInfo, - Filesystem, -} from './sandbox/filesystem' -export { - FileType, -} from './sandbox/filesystem' export { ProcessExitError } from './sandbox/process/processHandle' -export type { - ProcessResult, - Stdout, - Stderr, - PtyOutput, - ProcessHandle, -} from './sandbox/process/processHandle' +export type { ProcessHandle, ProcessResult, PtyOutput, Stderr, Stdout } from './sandbox/process/processHandle' export type { SandboxApiOpts } from './sandbox/sandboxApi' -export type { - ProcessInfo, - ProcessRequestOpts, - ProcessConnectOpts, - ProcessStartOpts, - Process, -} from './sandbox/process' +export type { Process, ProcessConnectOpts, ProcessInfo, ProcessRequestOpts, ProcessStartOpts } from './sandbox/process' export type { Pty } from './sandbox/pty' -export type { SandboxInfo } from './sandbox/sandboxApi' export type { SandboxOpts } from './sandbox' -import { Sandbox } from './sandbox' +export type { SandboxInfo } from './sandbox/sandboxApi' export { Sandbox } +import { Sandbox } from './sandbox' export default Sandbox diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 490ce91ba..639038888 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -8,6 +8,7 @@ import { EnvdApiClient } from '../../envd/api' import { Filesystem as FilesystemService } from '../../envd/filesystem/filesystem_connect' import { FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' +import { clearTimeout } from 'timers' import { FilesystemEvent, WatchHandle } from './watchHandle' export interface EntryInfo { @@ -80,8 +81,23 @@ export class Filesystem { return res.data } - async write(path: string, data: WriteData | { data: WriteData; filename: string }[], opts?: FilesystemRequestOpts): Promise { + async write(path: string, data: WriteData, opts?: FilesystemRequestOpts): Promise + async write(files: { data: WriteData; filename: string }[], opts?: FilesystemRequestOpts): Promise + async write(pathOrFiles: string | { data: WriteData; filename: string }[], dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts): Promise { + let path: string + let data: WriteData | { data: WriteData; filename: string }[] + + if (typeof pathOrFiles === 'string') { + path = pathOrFiles + data = dataOrOpts as WriteData + } else { + path = '' + data = pathOrFiles + opts = dataOrOpts as FilesystemRequestOpts + } + let blobs: Blob[] = [] + if (Array.isArray(data)) { for (const d of data) { const blob = await new Response(d.data).blob() @@ -251,11 +267,7 @@ export class Filesystem { } } - async watch( - path: string, - onEvent: (event: FilesystemEvent) => void | Promise, - opts?: FilesystemRequestOpts & { timeout?: number; onExit?: (err?: Error) => void | Promise } - ): Promise { + async watch(path: string, onEvent: (event: FilesystemEvent) => void | Promise, opts?: FilesystemRequestOpts & { timeout?: number; onExit?: (err?: Error) => void | Promise }): Promise { const requestTimeoutMs = opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs const controller = new AbortController() From 966f0c73afb4c58d6666f35cdb008f410786f714 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Fri, 4 Oct 2024 17:41:04 -0700 Subject: [PATCH 03/32] [WIP] Cleanup --- .../js-sdk/src/sandbox/filesystem/index.ts | 60 +++++-------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 639038888..739bba8f5 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -82,56 +82,30 @@ export class Filesystem { } async write(path: string, data: WriteData, opts?: FilesystemRequestOpts): Promise - async write(files: { data: WriteData; filename: string }[], opts?: FilesystemRequestOpts): Promise - async write(pathOrFiles: string | { data: WriteData; filename: string }[], dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts): Promise { - let path: string - let data: WriteData | { data: WriteData; filename: string }[] - - if (typeof pathOrFiles === 'string') { - path = pathOrFiles - data = dataOrOpts as WriteData - } else { - path = '' - data = pathOrFiles - opts = dataOrOpts as FilesystemRequestOpts - } - - let blobs: Blob[] = [] + async write(files: { path: string, data: WriteData }[], opts?: FilesystemRequestOpts): Promise + async write(pathOrFiles: string | { path: string; data: WriteData }[], dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts): Promise { + const { path, writeOpts, writeFiles } = typeof pathOrFiles === 'string' + ? { path: pathOrFiles, writeFiles: [{ data: dataOrOpts }], writeOpts: opts } + : { path: undefined, writeFiles: pathOrFiles, writeOpts: dataOrOpts } - if (Array.isArray(data)) { - for (const d of data) { - const blob = await new Response(d.data).blob() - blobs.push(blob) - } - } else { - const blob = await new Response(data).blob() - blobs.push(blob) - } + const blobs = await Promise.all(writeFiles.map(f => new Response(f.data).blob())) const res = await this.envdApi.api.POST('/files', { params: { query: { path, - username: opts?.user || defaultUsername, + username: writeOpts?.user || defaultUsername, }, }, bodySerializer() { - const fd = new FormData() - - for (let i = 0; i < blobs.length; i++) { - const blob = blobs[i] - if (Array.isArray(data)) { - const filename = data[i].filename - fd.append('file', blob, filename) - } else { - fd.append('file', blob) - } - } + return blobs.reduce((fd, blob, i) => { + fd.append('file', blob, writeFiles[i].path) - return fd + return fd + }, new FormData()) }, body: {}, - signal: this.connectionConfig.getSignal(opts?.requestTimeoutMs), + signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs), }) const err = await handleEnvdApiError(res) @@ -144,11 +118,7 @@ export class Filesystem { throw new Error('Expected to receive information about written file') } - if (files.length > 1) { - return files as EntryInfo[] - } else { - return files[0] as EntryInfo - } + return files.length === 1 ? files[0] : files } async list(path: string, opts?: FilesystemRequestOpts): Promise { @@ -274,8 +244,8 @@ export class Filesystem { const reqTimeout = requestTimeoutMs ? setTimeout(() => { - controller.abort() - }, requestTimeoutMs) + controller.abort() + }, requestTimeoutMs) : undefined const events = this.rpc.watchDir( From eda88d0c3464cb8df1d44dfd9384ae5b7f11bb33 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 7 Oct 2024 16:36:37 +0200 Subject: [PATCH 04/32] address PR comments --- .../js-sdk/src/sandbox/filesystem/index.ts | 69 ++++++++++++++----- .../js-sdk/tests/sandbox/files/write.test.ts | 48 ++++++++++--- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 739bba8f5..030e8b804 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -1,15 +1,22 @@ import { Code, ConnectError, createPromiseClient, PromiseClient, Transport } from '@connectrpc/connect' -import { ConnectionConfig, ConnectionOpts, defaultUsername, KEEPALIVE_PING_HEADER, KEEPALIVE_PING_INTERVAL_SEC, Username } from '../../connectionConfig' - -import { handleEnvdApiError, handleWatchDirStartEvent } from '../../envd/api' +import { + ConnectionConfig, + ConnectionOpts, + defaultUsername, + KEEPALIVE_PING_HEADER, + KEEPALIVE_PING_INTERVAL_SEC, + Username, +} from '../../connectionConfig' + +import { handleEnvdApiError } from '../../envd/api' import { authenticationHeader, handleRpcError } from '../../envd/rpc' import { EnvdApiClient } from '../../envd/api' import { Filesystem as FilesystemService } from '../../envd/filesystem/filesystem_connect' -import { FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' +import { EntryInfo, FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' -import { clearTimeout } from 'timers' import { FilesystemEvent, WatchHandle } from './watchHandle' +import { clearTimeout } from 'timers' export interface EntryInfo { name: string @@ -47,7 +54,11 @@ export class Filesystem { private readonly defaultWatchTimeout = 60_000 // 60 seconds - constructor(transport: Transport, private readonly envdApi: EnvdApiClient, private readonly connectionConfig: ConnectionConfig) { + constructor( + transport: Transport, + private readonly envdApi: EnvdApiClient, + private readonly connectionConfig: ConnectionConfig + ) { this.rpc = createPromiseClient(FilesystemService, transport) } @@ -55,7 +66,10 @@ export class Filesystem { async read(path: string, opts?: FilesystemRequestOpts & { format: 'bytes' }): Promise async read(path: string, opts?: FilesystemRequestOpts & { format: 'blob' }): Promise async read(path: string, opts?: FilesystemRequestOpts & { format: 'stream' }): Promise> - async read(path: string, opts?: FilesystemRequestOpts & { format?: 'text' | 'stream' | 'bytes' | 'blob' }): Promise { + async read( + path: string, + opts?: FilesystemRequestOpts & { format?: 'text' | 'stream' | 'bytes' | 'blob' } + ): Promise { const format = opts?.format ?? 'text' const res = await this.envdApi.api.GET('/files', { @@ -82,30 +96,43 @@ export class Filesystem { } async write(path: string, data: WriteData, opts?: FilesystemRequestOpts): Promise - async write(files: { path: string, data: WriteData }[], opts?: FilesystemRequestOpts): Promise - async write(pathOrFiles: string | { path: string; data: WriteData }[], dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts): Promise { - const { path, writeOpts, writeFiles } = typeof pathOrFiles === 'string' - ? { path: pathOrFiles, writeFiles: [{ data: dataOrOpts }], writeOpts: opts } - : { path: undefined, writeFiles: pathOrFiles, writeOpts: dataOrOpts } + async write(files: { path: string; data: WriteData }[], opts?: FilesystemRequestOpts): Promise + async write( + pathOrFiles: string | { path: string; data: WriteData }[], + dataOrOpts?: WriteData | FilesystemRequestOpts, + opts?: FilesystemRequestOpts + ): Promise { + if (typeof pathOrFiles === 'string' && Array.isArray(dataOrOpts)) { + throw new Error('Cannot specify path with array of files') + } + + const { path, writeOpts, writeFiles } = + typeof pathOrFiles === 'string' + ? { path: pathOrFiles, writeOpts: opts, writeFiles: [{ data: dataOrOpts }] } + : { path: undefined, writeOpts: dataOrOpts, writeFiles: pathOrFiles } - const blobs = await Promise.all(writeFiles.map(f => new Response(f.data).blob())) + const blobs = await Promise.all( + (writeFiles as { path: string; data: WriteData }[]).map((f) => new Response(f.data).blob()) + ) const res = await this.envdApi.api.POST('/files', { params: { query: { path, - username: writeOpts?.user || defaultUsername, + username: (writeOpts as FilesystemRequestOpts)?.user || defaultUsername, }, }, bodySerializer() { return blobs.reduce((fd, blob, i) => { + // Important: RFC 7578, Section 4.2 requires that if a filename is provided, + // the directory path information must not be used. fd.append('file', blob, writeFiles[i].path) return fd }, new FormData()) }, body: {}, - signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs), + signal: this.connectionConfig.getSignal((writeOpts as FilesystemRequestOpts)?.requestTimeoutMs), }) const err = await handleEnvdApiError(res) @@ -118,7 +145,7 @@ export class Filesystem { throw new Error('Expected to receive information about written file') } - return files.length === 1 ? files[0] : files + return files.length === 1 && path ? (files[0] as EntryInfo) : (files as EntryInfo[]) } async list(path: string, opts?: FilesystemRequestOpts): Promise { @@ -237,15 +264,19 @@ export class Filesystem { } } - async watch(path: string, onEvent: (event: FilesystemEvent) => void | Promise, opts?: FilesystemRequestOpts & { timeout?: number; onExit?: (err?: Error) => void | Promise }): Promise { + async watch( + path: string, + onEvent: (event: FilesystemEvent) => void | Promise, + opts?: FilesystemRequestOpts & { timeout?: number; onExit?: (err?: Error) => void | Promise } + ): Promise { const requestTimeoutMs = opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs const controller = new AbortController() const reqTimeout = requestTimeoutMs ? setTimeout(() => { - controller.abort() - }, requestTimeoutMs) + controller.abort() + }, requestTimeoutMs) : undefined const events = this.rpc.watchDir( diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 2e2649ac4..3e918cb26 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -21,19 +21,49 @@ sandboxTest('write file', async ({ sandbox }) => { }) sandboxTest('write multiple files', async ({ sandbox }) => { - let files: Array<{ data: WriteData; filename: string }> = [] + // Attempt to write with empty files array + await sandbox.files + .write([]) + .then((e) => { + assert.isUndefined(e) + }) + .catch((err) => { + assert.instanceOf(err, Error) + assert.include(err.message, 'Expected to receive information about written file') + }) + + // Attempt to write with patn and file array + await sandbox.files + .write('/path/to/file', [{ path: 'one_test_file.txt', data: 'This is a test file.' }]) + .then((e) => { + assert.isUndefined(e) + }) + .catch((err) => { + assert.instanceOf(err, Error) + assert.include(err.message, 'Cannot specify path with array of files') + }) + + // Attempt to write with one file in array + const info = await sandbox.files.write([{ path: 'one_test_file.txt', data: 'This is a test file.' }]) + assert.isTrue(Array.isArray(info)) + assert.equal(info[0].name, 'one_test_file.txt') + assert.equal(info[0].type, 'file') + assert.equal(info[0].path, `/home/user/one_test_file.txt`) + + // Attempt to write with multiple files in array + let files: Array<{ data: WriteData; path: string }> = [] for (let i = 0; i < 10; i++) { - const filename = `multi_test_file${i}.txt` - onTestFinished(async () => await sandbox.files.remove(filename)) + const path = `multi_test_file${i}.txt` + onTestFinished(async () => await sandbox.files.remove(path)) files.push({ - filename: `multi_test_file${i}.txt`, + path: `multi_test_file${i}.txt`, data: `This is a test file ${i}.`, }) } - const infos = await sandbox.files.write('', files) + const infos = await sandbox.files.write(files) assert.isTrue(Array.isArray(infos)) assert.equal((infos as EntryInfo[]).length, files.length) @@ -42,13 +72,13 @@ sandboxTest('write multiple files', async ({ sandbox }) => { const file = files[i] const info = infos[i] as EntryInfo - assert.equal(info.name, file.filename) + assert.equal(info.name, file.path) assert.equal(info.type, 'file') - assert.equal(info.path, `/home/user/${file.filename}`) + assert.equal(info.path, `/home/user/${file.path}`) - const exists = await sandbox.files.exists(file.filename) + const exists = await sandbox.files.exists(file.path) assert.isTrue(exists) - const readContent = await sandbox.files.read(file.filename) + const readContent = await sandbox.files.read(file.path) assert.equal(readContent, file.data) } }) From a2d4ad72b6505e44dca9ecb2e7279c486564409c Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 7 Oct 2024 16:47:00 +0200 Subject: [PATCH 05/32] boyscouting: fix some docstrings --- packages/python-sdk/e2b/exceptions.py | 8 ++++---- packages/python-sdk/e2b/sandbox_async/main.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index d785367ab..42db803b4 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -30,10 +30,10 @@ class TimeoutException(SandboxException): """ Raised when a timeout occurs. - The [unavailable] exception type is caused by sandbox timeout.\n - The [canceled] exception type is caused by exceeding request timeout.\n - The [deadline_exceeded] exception type is caused by exceeding the timeout for process, watch, etc.\n - The [unknown] exception type is sometimes caused by the sandbox timeout when the request is not processed correctly.\n + The `unavailable` exception type is caused by sandbox timeout.\n + The `canceled` exception type is caused by exceeding request timeout.\n + The `deadline_exceeded` exception type is caused by exceeding the timeout for process, watch, etc.\n + The `unknown` exception type is sometimes caused by the sandbox timeout when the request is not processed correctly.\n """ pass diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py index e3c3d4cc3..7b80f5c10 100644 --- a/packages/python-sdk/e2b/sandbox_async/main.py +++ b/packages/python-sdk/e2b/sandbox_async/main.py @@ -46,6 +46,7 @@ def commands(self) -> Process: @property def pty(self) -> Pty: + """Get a Pty Object""" return self._pty @property From a379a587f060fda7ca10004a4b0427308f15bd35 Mon Sep 17 00:00:00 2001 From: 0div Date: Mon, 7 Oct 2024 21:10:28 +0200 Subject: [PATCH 06/32] Add multi-file write support for python-sdk sync --- .../js-sdk/tests/sandbox/files/write.test.ts | 3 +- .../e2b/sandbox/filesystem/filesystem.py | 9 ++- .../e2b/sandbox_sync/filesystem/filesystem.py | 55 ++++++++++++++----- .../sync/sandbox_sync/files/test_write.py | 46 ++++++++++++++++ 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 3e918cb26..137e84ebc 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -32,7 +32,7 @@ sandboxTest('write multiple files', async ({ sandbox }) => { assert.include(err.message, 'Expected to receive information about written file') }) - // Attempt to write with patn and file array + // Attempt to write with path and file array await sandbox.files .write('/path/to/file', [{ path: 'one_test_file.txt', data: 'This is a test file.' }]) .then((e) => { @@ -68,6 +68,7 @@ sandboxTest('write multiple files', async ({ sandbox }) => { assert.isTrue(Array.isArray(infos)) assert.equal((infos as EntryInfo[]).length, files.length) + // Attempt to write with multiple files in array for (let i = 0; i < files.length; i++) { const file = files[i] const info = infos[i] as EntryInfo diff --git a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py index 8188f144b..8d5eb1815 100644 --- a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py @@ -1,6 +1,6 @@ from enum import Enum from dataclasses import dataclass -from typing import Optional +from typing import IO, Optional, Union from e2b.envd.filesystem import filesystem_pb2 @@ -22,3 +22,10 @@ class EntryInfo: name: str type: Optional[FileType] path: str + +WriteData = Union[str, bytes, IO] + +@dataclass +class FileWriteData: + path: str + data: WriteData diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index eea910554..401f00e01 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,5 +1,6 @@ from io import TextIOBase -from typing import IO, Iterator, List, Literal, Optional, Union, overload +from typing import Iterator, List, Literal, Optional, overload +from e2b.sandbox.filesystem.filesystem import WriteData, FileWriteData import e2b_connect import httpcore @@ -93,23 +94,48 @@ def read( def write( self, - path: str, - data: Union[str, bytes, IO], + path: str | None, + data: WriteData | List[FileWriteData], user: Username = "user", request_timeout: Optional[float] = None, - ) -> EntryInfo: - """Write to file + ) -> EntryInfo | List[EntryInfo]: + """Write to file(s) When writing to a file that doesn't exist, the file will get created. When writing to a file that already exists, the file will get overwritten. When writing to a file that's in a directory that doesn't exist, you'll get an error. """ - if isinstance(data, TextIOBase): - data = data.read().encode() + + write_files = [] + if path is None: + if not isinstance(data, list): + raise Exception("Expected data to be a list of FileWriteData") + write_files = data + else: + if isinstance(data, list): + raise Exception("Cannot specify path with array of files") + write_files = [{"path": path, "data": data}] + + if len(write_files) == 0: + raise Exception("Need at least one file to write") + + # Prepare the files for the multipart/form-data request + httpx_files = [] + for file in write_files: + file_path, file_data = file['path'], file['data'] + if isinstance(file_data, str) or isinstance(file_data, bytes): + httpx_files.append(('file', (file_path, file_data))) + elif isinstance(file_data, TextIOBase): + httpx_files.append(('file', (file_path, file_data.read()))) + else: + raise ValueError(f"Unsupported data type for file {file_path}") + + params = {"username": user} + if path is not None: params["path"] = path r = self._envd_api.post( ENVD_API_FILES_ROUTE, - files={"file": data}, - params={"path": path, "username": user}, + files=httpx_files, + params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) @@ -117,13 +143,16 @@ def write( if err: raise err - files = r.json() + write_files = r.json() - if not isinstance(files, list) or len(files) == 0: + if not isinstance(write_files, list) or len(write_files) == 0: raise Exception("Expected to receive information about written file") - file = files[0] - return EntryInfo(**file) + if len(write_files) == 1 and path: + file = write_files[0] + return EntryInfo(**file) + else: + return [EntryInfo(**file) for file in write_files] def list( self, diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index ba0390ee1..83addec27 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -1,3 +1,5 @@ +from e2b.sandbox.filesystem.filesystem import EntryInfo + def test_write_file(sandbox): filename = "test_write.txt" content = "This is a test file." @@ -11,6 +13,50 @@ def test_write_file(sandbox): read_content = sandbox.files.read(filename) assert read_content == content +def test_write_multiple_files(sandbox): + # Attempt to write with empty files array + try: + sandbox.files.write(None, []) + except Exception as e: + assert "Need at least one file to write" in str(e) + + # Attempt to write with path and file array + try: + sandbox.files.write("/path/to/file", [{ "path": "one_test_file.txt", "data": "This is a test file." }]) + except Exception as e: + assert "Cannot specify path with array of files" in str(e) + + # Attempt to write with one file in array + info = sandbox.files.write(None, [{ "path": "one_test_file.txt", "data": "This is a test file." }]) + assert isinstance(info, list) + assert len(info) == 1 + info = info[0] + assert isinstance(info, EntryInfo) + assert info.path == "/home/user/one_test_file.txt" + exists = sandbox.files.exists(info.path) + assert exists + + read_content = sandbox.files.read(info.path) + assert read_content == "This is a test file." + + # Attempt to write with multiple files in array + files = [] + for i in range(10): + filename = f"test_write_{i}.txt" + content = f"This is a test file {i}." + files.append({"path": filename, "data": content}) + + infos = sandbox.files.write(None, files) + assert isinstance(infos, list) + assert len(infos) == len(files) + for i, info in enumerate(infos): + assert isinstance(info, EntryInfo) + assert info.path == f"/home/user/test_write_{i}.txt" + exists = sandbox.files.exists(filename) + assert exists + + read_content = sandbox.files.read(info.path) + assert read_content == files[i]["data"] def test_overwrite_file(sandbox): filename = "test_overwrite.txt" From 608ef45fa5178715da6420a8fe19bfb1d3186593 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Oct 2024 12:32:33 +0200 Subject: [PATCH 07/32] Use `@overload` --- .../e2b/sandbox/filesystem/filesystem.py | 2 +- .../e2b/sandbox_sync/filesystem/filesystem.py | 44 ++++++++++++++----- .../sync/sandbox_sync/files/test_write.py | 6 +-- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py index 8d5eb1815..4d5037684 100644 --- a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py @@ -26,6 +26,6 @@ class EntryInfo: WriteData = Union[str, bytes, IO] @dataclass -class FileWriteData: +class WriteEntry: path: str data: WriteData diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 401f00e01..b7bfca494 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,6 +1,6 @@ from io import TextIOBase from typing import Iterator, List, Literal, Optional, overload -from e2b.sandbox.filesystem.filesystem import WriteData, FileWriteData +from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry import e2b_connect import httpcore @@ -92,12 +92,32 @@ def read( elif format == "stream": return r.iter_bytes() + @overload def write( self, - path: str | None, - data: WriteData | List[FileWriteData], + path: str, + data: WriteData, user: Username = "user", request_timeout: Optional[float] = None, + ) -> EntryInfo: + """Write to file""" + + @overload + def write( + self, + files: List[WriteEntry], + user: Optional[Username] = "user", + path: Optional[str] = None, + request_timeout: Optional[float] = None, + ) -> List[EntryInfo]: + """Write multiple files""" + + def write( + self, + path_or_files: str | List[WriteEntry], + data_or_user: WriteData | Username = "user", + user_or_request_timeout: Optional[float | Username] = None, + request_timeout_or_none: Optional[float] = None ) -> EntryInfo | List[EntryInfo]: """Write to file(s) When writing to a file that doesn't exist, the file will get created. @@ -105,16 +125,16 @@ def write( When writing to a file that's in a directory that doesn't exist, you'll get an error. """ - write_files = [] - if path is None: - if not isinstance(data, list): - raise Exception("Expected data to be a list of FileWriteData") - write_files = data - else: - if isinstance(data, list): + path, write_files, user, request_timeout, = None, [], "user", None + if isinstance(path_or_files, str): + if isinstance(data_or_user, list): raise Exception("Cannot specify path with array of files") - write_files = [{"path": path, "data": data}] - + path, write_files, user, request_timeout = \ + path_or_files, [{"path": path_or_files, "data": data_or_user}], user_or_request_timeout or "user", request_timeout_or_none + else: + path, write_files, user, request_timeout = \ + None, path_or_files, data_or_user, user_or_request_timeout + if len(write_files) == 0: raise Exception("Need at least one file to write") diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index 83addec27..3ccbd41e0 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -16,7 +16,7 @@ def test_write_file(sandbox): def test_write_multiple_files(sandbox): # Attempt to write with empty files array try: - sandbox.files.write(None, []) + sandbox.files.write([]) except Exception as e: assert "Need at least one file to write" in str(e) @@ -27,7 +27,7 @@ def test_write_multiple_files(sandbox): assert "Cannot specify path with array of files" in str(e) # Attempt to write with one file in array - info = sandbox.files.write(None, [{ "path": "one_test_file.txt", "data": "This is a test file." }]) + info = sandbox.files.write([{ "path": "one_test_file.txt", "data": "This is a test file." }]) assert isinstance(info, list) assert len(info) == 1 info = info[0] @@ -46,7 +46,7 @@ def test_write_multiple_files(sandbox): content = f"This is a test file {i}." files.append({"path": filename, "data": content}) - infos = sandbox.files.write(None, files) + infos = sandbox.files.write(files) assert isinstance(infos, list) assert len(infos) == len(files) for i, info in enumerate(infos): From f010211787441284c037c795a4fc57641f4ebc10 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Oct 2024 17:15:13 +0200 Subject: [PATCH 08/32] adapt multi file write tests for nested dirs --- packages/js-sdk/src/sandbox/filesystem/index.ts | 2 +- packages/js-sdk/tests/sandbox/files/write.test.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index b71ab6184..b4b06a44c 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -13,7 +13,7 @@ import { authenticationHeader, handleRpcError } from '../../envd/rpc' import { EnvdApiClient } from '../../envd/api' import { Filesystem as FilesystemService } from '../../envd/filesystem/filesystem_connect' -import { EntryInfo, FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' +import { FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' import { clearTimeout } from 'timers' import { FilesystemEvent, WatchHandle } from './watchHandle' diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 137e84ebc..1ad4249da 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -1,3 +1,4 @@ +import path from 'path' import { assert, onTestFinished } from 'vitest' import { EntryInfo } from '../../../src/index.js' @@ -54,11 +55,17 @@ sandboxTest('write multiple files', async ({ sandbox }) => { let files: Array<{ data: WriteData; path: string }> = [] for (let i = 0; i < 10; i++) { - const path = `multi_test_file${i}.txt` + let path = '' + if (i % 2 == 0) { + path = `/${i}/multi_test_file${i}.txt` + } else { + path = `/home/user/multi_test_file${i}.txt` + } + onTestFinished(async () => await sandbox.files.remove(path)) files.push({ - path: `multi_test_file${i}.txt`, + path: path, data: `This is a test file ${i}.`, }) } @@ -73,9 +80,9 @@ sandboxTest('write multiple files', async ({ sandbox }) => { const file = files[i] const info = infos[i] as EntryInfo - assert.equal(info.name, file.path) + assert.equal(info.name, path.basename(file.path)) + assert.equal(info.path, file.path) assert.equal(info.type, 'file') - assert.equal(info.path, `/home/user/${file.path}`) const exists = await sandbox.files.exists(file.path) assert.isTrue(exists) From fbc4af4908eaff58f06b7782577a78899e7f11c6 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Oct 2024 18:23:46 +0200 Subject: [PATCH 09/32] allow passing empty array of files in python-sdk --- .../python-sdk/e2b/sandbox_sync/filesystem/filesystem.py | 8 ++++---- .../tests/sync/sandbox_sync/files/test_write.py | 7 +++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 7af797ba9..f0b55b529 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -155,7 +155,7 @@ def write( When writing to a file that's in a directory that doesn't exist, you'll get an error. """ - path, write_files, user, request_timeout, = None, [], "user", None + path, write_files, user, request_timeout = None, [], "user", None if isinstance(path_or_files, str): if isinstance(data_or_user, list): raise Exception("Cannot specify path with array of files") @@ -165,9 +165,6 @@ def write( path, write_files, user, request_timeout = \ None, path_or_files, data_or_user, user_or_request_timeout - if len(write_files) == 0: - raise Exception("Need at least one file to write") - # Prepare the files for the multipart/form-data request httpx_files = [] for file in write_files: @@ -178,6 +175,9 @@ def write( httpx_files.append(('file', (file_path, file_data.read()))) else: raise ValueError(f"Unsupported data type for file {file_path}") + + # Allow passing empty list of files + if len(httpx_files) == 0: return [] params = {"username": user} if path is not None: params["path"] = path diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index 3ccbd41e0..a9bbf419d 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -15,10 +15,9 @@ def test_write_file(sandbox): def test_write_multiple_files(sandbox): # Attempt to write with empty files array - try: - sandbox.files.write([]) - except Exception as e: - assert "Need at least one file to write" in str(e) + empty_info = sandbox.files.write([]) + assert isinstance(empty_info, list) + assert len(empty_info) == 0 # Attempt to write with path and file array try: From 2afb6d1ef46198bd34669c1446a974d962ffb4d1 Mon Sep 17 00:00:00 2001 From: 0div Date: Tue, 8 Oct 2024 18:24:20 +0200 Subject: [PATCH 10/32] allow passing empty array of files in js-sdk --- packages/js-sdk/src/sandbox/filesystem/index.ts | 2 +- packages/js-sdk/tests/sandbox/files/write.test.ts | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index b4b06a44c..f805c5d4a 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -171,7 +171,7 @@ export class Filesystem { } const files = res.data - if (!files || files.length === 0) { + if (!files) { throw new Error('Expected to receive information about written file') } diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 1ad4249da..2aa82d1e0 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -23,18 +23,13 @@ sandboxTest('write file', async ({ sandbox }) => { sandboxTest('write multiple files', async ({ sandbox }) => { // Attempt to write with empty files array - await sandbox.files - .write([]) - .then((e) => { - assert.isUndefined(e) - }) - .catch((err) => { - assert.instanceOf(err, Error) - assert.include(err.message, 'Expected to receive information about written file') - }) + const emptyInfo = await sandbox.files.write([]) + assert.isTrue(Array.isArray(emptyInfo)) + assert.equal(emptyInfo.length, 0) // Attempt to write with path and file array await sandbox.files + // @ts-ignore .write('/path/to/file', [{ path: 'one_test_file.txt', data: 'This is a test file.' }]) .then((e) => { assert.isUndefined(e) From 68e5efac41923e9286d4b8f7c8cc0f8987178343 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 9 Oct 2024 11:19:53 +0200 Subject: [PATCH 11/32] address PR comments --- packages/js-sdk/src/errors.ts | 18 +++++------ .../js-sdk/src/sandbox/filesystem/index.ts | 32 ++++++++++++------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index f3b016596..9e26f74c6 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -7,7 +7,7 @@ export function formatSandboxTimeoutError(message: string) { /** * Thrown when a sandbox error occurs. - * + * * Base class for all sandbox errors. */ export class SandboxError extends Error { @@ -19,14 +19,14 @@ export class SandboxError extends Error { /** * Thrown when a timeout error occurs. - * - * The [unavailable] error type is caused by sandbox timeout. - * - * The [canceled] error type is caused by exceeding request timeout. - * - * The [deadline_exceeded] error type is caused by exceeding the timeout for process, watch, etc. - * - * The [unknown] error type is sometimes caused by the sandbox timeout when the request is not processed correctly. + * + * The `unavailable` error type is caused by sandbox timeout. + * + * The `canceled` error type is caused by exceeding request timeout. + * + * The `deadline_exceeded` error type is caused by exceeding the timeout for process, watch, etc. + * + * The `unknown` error type is sometimes caused by the sandbox timeout when the request is not processed correctly. */ export class TimeoutError extends SandboxError { constructor(message: string) { diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index f805c5d4a..e2516dd30 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -15,7 +15,6 @@ import { EnvdApiClient } from '../../envd/api' import { Filesystem as FilesystemService } from '../../envd/filesystem/filesystem_connect' import { FileType as FsFileType } from '../../envd/filesystem/filesystem_pb' -import { clearTimeout } from 'timers' import { FilesystemEvent, WatchHandle } from './watchHandle' /** @@ -37,6 +36,11 @@ export const enum FileType { export type WriteData = string | ArrayBuffer | Blob | ReadableStream +export type WriteEntry = { + path: string + data: WriteData +} + function mapFileType(fileType: FsFileType) { switch (fileType) { case FsFileType.DIRECTORY: @@ -126,9 +130,9 @@ export class Filesystem { } async write(path: string, data: WriteData, opts?: FilesystemRequestOpts): Promise - async write(files: { path: string; data: WriteData }[], opts?: FilesystemRequestOpts): Promise + async write(files: WriteEntry[], opts?: FilesystemRequestOpts): Promise async write( - pathOrFiles: string | { path: string; data: WriteData }[], + pathOrFiles: string | WriteEntry[], dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts ): Promise { @@ -138,31 +142,35 @@ export class Filesystem { const { path, writeOpts, writeFiles } = typeof pathOrFiles === 'string' - ? { path: pathOrFiles, writeOpts: opts, writeFiles: [{ data: dataOrOpts }] } - : { path: undefined, writeOpts: dataOrOpts, writeFiles: pathOrFiles } + ? { + path: pathOrFiles, + writeOpts: opts as FilesystemRequestOpts, + writeFiles: [{ data: dataOrOpts as WriteData }], + } + : { path: undefined, writeOpts: dataOrOpts as FilesystemRequestOpts, writeFiles: pathOrFiles as WriteEntry[] } - const blobs = await Promise.all( - (writeFiles as { path: string; data: WriteData }[]).map((f) => new Response(f.data).blob()) - ) + const blobs = await Promise.all(writeFiles.map((f) => new Response(f.data).blob())) const res = await this.envdApi.api.POST('/files', { params: { query: { path, - username: (writeOpts as FilesystemRequestOpts)?.user || defaultUsername, + username: writeOpts?.user || defaultUsername, }, }, bodySerializer() { return blobs.reduce((fd, blob, i) => { // Important: RFC 7578, Section 4.2 requires that if a filename is provided, // the directory path information must not be used. + // BUT in our case we need to use the directory path information with a custom + // muktipart part name getter in envd. fd.append('file', blob, writeFiles[i].path) return fd }, new FormData()) }, body: {}, - signal: this.connectionConfig.getSignal((writeOpts as FilesystemRequestOpts)?.requestTimeoutMs), + signal: this.connectionConfig.getSignal(writeOpts?.requestTimeoutMs), }) const err = await handleEnvdApiError(res) @@ -170,12 +178,12 @@ export class Filesystem { throw err } - const files = res.data + const files = res.data as EntryInfo[] if (!files) { throw new Error('Expected to receive information about written file') } - return files.length === 1 && path ? (files[0] as EntryInfo) : (files as EntryInfo[]) + return files.length === 1 && path ? files[0] : files } /** From 365af432f1068dc5cebc1ce191ae340f36f2e757 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 9 Oct 2024 12:29:59 +0200 Subject: [PATCH 12/32] add extra tests to sandbox_sync write --- .../tests/sync/sandbox_sync/files/test_write.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index a9bbf419d..22bba6249 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -4,6 +4,12 @@ def test_write_file(sandbox): filename = "test_write.txt" content = "This is a test file." + # Attempt to write without path + try: + sandbox.files.write(None, content) + except Exception as e: + assert "object is not iterable" in str(e) + info = sandbox.files.write(filename, content) assert info.path == f"/home/user/{filename}" @@ -19,6 +25,12 @@ def test_write_multiple_files(sandbox): assert isinstance(empty_info, list) assert len(empty_info) == 0 + # Attempt to write with None path and empty files array + try: + sandbox.files.write(None, []) + except Exception as e: + assert "object is not iterable" in str(e) + # Attempt to write with path and file array try: sandbox.files.write("/path/to/file", [{ "path": "one_test_file.txt", "data": "This is a test file." }]) From 7767d8f2ee46f202e8add061fb9ea166cea1b9df Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 9 Oct 2024 13:37:05 +0200 Subject: [PATCH 13/32] updated js-sdk tests to check empty path behavior --- .../js-sdk/tests/sandbox/files/write.test.ts | 37 +++++++++++++++---- .../sync/sandbox_sync/files/test_write.py | 6 +-- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 2aa82d1e0..51a7b968b 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -1,15 +1,26 @@ import path from 'path' import { assert, onTestFinished } from 'vitest' -import { EntryInfo } from '../../../src/index.js' -import { WriteData } from '../../../src/sandbox/filesystem/index.js' +import { WriteEntry } from '../../../src/sandbox/filesystem/index.js' import { sandboxTest } from '../../setup.js' sandboxTest('write file', async ({ sandbox }) => { const filename = 'test_write.txt' const content = 'This is a test file.' - const info = (await sandbox.files.write(filename, content)) as EntryInfo + // Attempt to write with undefined path and content + await sandbox.files + // @ts-ignore + .write(undefined, content) + .then((e) => { + assert.isUndefined(e) + }) + .catch((err) => { + assert.instanceOf(err, Error) + assert.include(err.message, 'Cannot read properties of undefined') + }) + + const info = await sandbox.files.write(filename, content) assert.isFalse(Array.isArray(info)) assert.equal(info.name, filename) assert.equal(info.type, 'file') @@ -27,6 +38,18 @@ sandboxTest('write multiple files', async ({ sandbox }) => { assert.isTrue(Array.isArray(emptyInfo)) assert.equal(emptyInfo.length, 0) + // Attempt to write with undefined path and file array + await sandbox.files + // @ts-ignore + .write(undefined, [{ path: 'one_test_file.txt', data: 'This is a test file.' }]) + .then((e) => { + assert.isUndefined(e) + }) + .catch((err) => { + assert.instanceOf(err, Error) + assert.include(err.message, 'Cannot read properties of undefined') + }) + // Attempt to write with path and file array await sandbox.files // @ts-ignore @@ -47,7 +70,7 @@ sandboxTest('write multiple files', async ({ sandbox }) => { assert.equal(info[0].path, `/home/user/one_test_file.txt`) // Attempt to write with multiple files in array - let files: Array<{ data: WriteData; path: string }> = [] + let files: WriteEntry[] = [] for (let i = 0; i < 10; i++) { let path = '' @@ -68,12 +91,12 @@ sandboxTest('write multiple files', async ({ sandbox }) => { const infos = await sandbox.files.write(files) assert.isTrue(Array.isArray(infos)) - assert.equal((infos as EntryInfo[]).length, files.length) + assert.equal(infos.length, files.length) // Attempt to write with multiple files in array for (let i = 0; i < files.length; i++) { const file = files[i] - const info = infos[i] as EntryInfo + const info = infos[i] assert.equal(info.name, path.basename(file.path)) assert.equal(info.path, file.path) @@ -90,7 +113,7 @@ sandboxTest('write file', async ({ sandbox }) => { const filename = 'test_write.txt' const content = 'This is a test file.' - const info = (await sandbox.files.write(filename, content)) as EntryInfo + const info = await sandbox.files.write(filename, content) assert.isFalse(Array.isArray(info)) assert.equal(info.name, filename) assert.equal(info.type, 'file') diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index 22bba6249..0a97b542b 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -53,9 +53,9 @@ def test_write_multiple_files(sandbox): # Attempt to write with multiple files in array files = [] for i in range(10): - filename = f"test_write_{i}.txt" + path = f"test_write_{i}.txt" content = f"This is a test file {i}." - files.append({"path": filename, "data": content}) + files.append({"path": path, "data": content}) infos = sandbox.files.write(files) assert isinstance(infos, list) @@ -63,7 +63,7 @@ def test_write_multiple_files(sandbox): for i, info in enumerate(infos): assert isinstance(info, EntryInfo) assert info.path == f"/home/user/test_write_{i}.txt" - exists = sandbox.files.exists(filename) + exists = sandbox.files.exists(path) assert exists read_content = sandbox.files.read(info.path) From f5cd1c07675340de6ae053dfe673291ebb890c85 Mon Sep 17 00:00:00 2001 From: 0div Date: Wed, 9 Oct 2024 14:07:14 +0200 Subject: [PATCH 14/32] add multifile upload to sanbox_async --- .../sandbox_async/filesystem/filesystem.py | 80 ++++++++++++++++--- .../async/sandbox_async/files/test_write.py | 57 ++++++++++++- 2 files changed, 126 insertions(+), 11 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index 6a47433af..8d7fc77c9 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,5 +1,6 @@ from io import TextIOBase -from typing import IO, AsyncIterator, List, Literal, Optional, Union, overload +from typing import AsyncIterator, List, Literal, Optional, overload +from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry import e2b_connect as connect import httpcore @@ -105,10 +106,11 @@ async def read( elif format == "stream": return r.aiter_bytes() + @overload async def write( self, path: str, - data: Union[str, bytes, IO], + data: WriteData, user: Username = "user", request_timeout: Optional[float] = None, ) -> EntryInfo: @@ -124,13 +126,67 @@ async def write( :param request_timeout: Timeout for the request :return: Information about the written file """ - if isinstance(data, TextIOBase): - data = data.read().encode() + + @overload + async def write( + self, + files: List[WriteEntry], + user: Optional[Username] = "user", + request_timeout: Optional[float] = None, + ) -> List[EntryInfo]: + """ + Writes multiple files. + + :param files: list of files to write + :param user: Run the operation as this user + :param request_timeout: Timeout for the request + :return: Information about the written files + """ + + async def write( + self, + path_or_files: str | List[WriteEntry], + data_or_user: WriteData | Username = "user", + user_or_request_timeout: Optional[float | Username] = None, + request_timeout_or_none: Optional[float] = None + ) -> EntryInfo | List[EntryInfo]: + """ + Writes content to a file on the path. + When writing to a file that doesn't exist, the file will get created. + When writing to a file that already exists, the file will get overwritten. + When writing to a file that's in a directory that doesn't exist, you'll get an error. + """ + path, write_files, user, request_timeout = None, [], "user", None + if isinstance(path_or_files, str): + if isinstance(data_or_user, list): + raise Exception("Cannot specify path with array of files") + path, write_files, user, request_timeout = \ + path_or_files, [{"path": path_or_files, "data": data_or_user}], user_or_request_timeout or "user", request_timeout_or_none + else: + path, write_files, user, request_timeout = \ + None, path_or_files, data_or_user, user_or_request_timeout + + # Prepare the files for the multipart/form-data request + httpx_files = [] + for file in write_files: + file_path, file_data = file['path'], file['data'] + if isinstance(file_data, str) or isinstance(file_data, bytes): + httpx_files.append(('file', (file_path, file_data))) + elif isinstance(file_data, TextIOBase): + httpx_files.append(('file', (file_path, file_data.read()))) + else: + raise ValueError(f"Unsupported data type for file {file_path}") + + # Allow passing empty list of files + if len(httpx_files) == 0: return [] + + params = {"username": user} + if path is not None: params["path"] = path r = await self._envd_api.post( ENVD_API_FILES_ROUTE, - files={"file": data}, - params={"path": path, "username": user}, + files=httpx_files, + params=params, timeout=self._connection_config.get_request_timeout(request_timeout), ) @@ -138,13 +194,17 @@ async def write( if err: raise err - files = r.json() + write_files = r.json() - if not isinstance(files, list) or len(files) == 0: + if not isinstance(write_files, list) or len(write_files) == 0: raise Exception("Expected to receive information about written file") - file = files[0] - return EntryInfo(**file) + if len(write_files) == 1 and path: + file = write_files[0] + return EntryInfo(**file) + else: + return [EntryInfo(**file) for file in write_files] + async def list( self, diff --git a/packages/python-sdk/tests/async/sandbox_async/files/test_write.py b/packages/python-sdk/tests/async/sandbox_async/files/test_write.py index bdfbf514a..8d388a93d 100644 --- a/packages/python-sdk/tests/async/sandbox_async/files/test_write.py +++ b/packages/python-sdk/tests/async/sandbox_async/files/test_write.py @@ -1,10 +1,16 @@ from e2b import AsyncSandbox - +from e2b.sandbox_async.filesystem.filesystem import EntryInfo async def test_write_file(async_sandbox: AsyncSandbox): filename = "test_write.txt" content = "This is a test file." + # Attempt to write without path + try: + await async_sandbox.files.write(None, content) + except Exception as e: + assert "object is not iterable" in str(e) + info = await async_sandbox.files.write(filename, content) assert info.path == f"/home/user/{filename}" @@ -14,6 +20,55 @@ async def test_write_file(async_sandbox: AsyncSandbox): read_content = await async_sandbox.files.read(filename) assert read_content == content +async def test_write_multiple_files(async_sandbox: AsyncSandbox): + # Attempt to write with empty files array + empty_info = await async_sandbox.files.write([]) + assert isinstance(empty_info, list) + assert len(empty_info) == 0 + + # Attempt to write with None path and empty files array + try: + await async_sandbox.files.write(None, []) + except Exception as e: + assert "object is not iterable" in str(e) + + # Attempt to write with path and file array + try: + await async_sandbox.files.write("/path/to/file", [{ "path": "one_test_file.txt", "data": "This is a test file." }]) + except Exception as e: + assert "Cannot specify path with array of files" in str(e) + + # Attempt to write with one file in array + info = await async_sandbox.files.write([{ "path": "one_test_file.txt", "data": "This is a test file." }]) + assert isinstance(info, list) + assert len(info) == 1 + info = info[0] + assert isinstance(info, EntryInfo) + assert info.path == "/home/user/one_test_file.txt" + exists = await async_sandbox.files.exists(info.path) + assert exists + + read_content = await async_sandbox.files.read(info.path) + assert read_content == "This is a test file." + + # Attempt to write with multiple files in array + files = [] + for i in range(10): + path = f"test_write_{i}.txt" + content = f"This is a test file {i}." + files.append({"path": path, "data": content}) + + infos = await async_sandbox.files.write(files) + assert isinstance(infos, list) + assert len(infos) == len(files) + for i, info in enumerate(infos): + assert isinstance(info, EntryInfo) + assert info.path == f"/home/user/test_write_{i}.txt" + exists = await async_sandbox.files.exists(path) + assert exists + + read_content = await async_sandbox.files.read(info.path) + assert read_content == files[i]["data"] async def test_overwrite_file(async_sandbox: AsyncSandbox): filename = "test_overwrite.txt" From 2956a6e7b92f22d7cecbffc0d602e075598a28b1 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 10 Oct 2024 12:04:42 +0200 Subject: [PATCH 15/32] better error messages in python-sdk --- .../e2b/sandbox_async/filesystem/filesystem.py | 4 +++- .../e2b/sandbox_sync/filesystem/filesystem.py | 16 +++++++--------- .../async/sandbox_async/files/test_write.py | 6 +++--- .../tests/sync/sandbox_sync/files/test_write.py | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index f9ede3c29..720b7c74d 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -159,10 +159,12 @@ async def write( path, write_files, user, request_timeout = None, [], "user", None if isinstance(path_or_files, str): if isinstance(data_or_user, list): - raise Exception("Cannot specify path with array of files") + raise Exception("Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files.") path, write_files, user, request_timeout = \ path_or_files, [{"path": path_or_files, "data": data_or_user}], user_or_request_timeout or "user", request_timeout_or_none else: + if path_or_files is None: + raise Exception("Path or files are required") path, write_files, user, request_timeout = \ None, path_or_files, data_or_user, user_or_request_timeout diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 94b7e144d..015d9e8c2 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -133,7 +133,10 @@ def write( request_timeout: Optional[float] = None, ) -> List[EntryInfo]: """ - Writes multiple files. + Writes a list of files to the filesystem. + When writing to a file that doesn't exist, the file will get created. + When writing to a file that already exists, the file will get overwritten. + When writing to a file that's in a directory that doesn't exist, you'll get an error. :param files: list of files to write :param user: Run the operation as this user @@ -148,20 +151,15 @@ def write( user_or_request_timeout: Optional[float | Username] = None, request_timeout_or_none: Optional[float] = None ) -> EntryInfo | List[EntryInfo]: - """ - Writes content to a file on the path. - When writing to a file that doesn't exist, the file will get created. - When writing to a file that already exists, the file will get overwritten. - When writing to a file that's in a directory that doesn't exist, you'll get an error. - """ - path, write_files, user, request_timeout = None, [], "user", None if isinstance(path_or_files, str): if isinstance(data_or_user, list): - raise Exception("Cannot specify path with array of files") + raise Exception("Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files.") path, write_files, user, request_timeout = \ path_or_files, [{"path": path_or_files, "data": data_or_user}], user_or_request_timeout or "user", request_timeout_or_none else: + if path_or_files is None: + raise Exception("Path or files are required") path, write_files, user, request_timeout = \ None, path_or_files, data_or_user, user_or_request_timeout diff --git a/packages/python-sdk/tests/async/sandbox_async/files/test_write.py b/packages/python-sdk/tests/async/sandbox_async/files/test_write.py index 8d388a93d..b897f3673 100644 --- a/packages/python-sdk/tests/async/sandbox_async/files/test_write.py +++ b/packages/python-sdk/tests/async/sandbox_async/files/test_write.py @@ -9,7 +9,7 @@ async def test_write_file(async_sandbox: AsyncSandbox): try: await async_sandbox.files.write(None, content) except Exception as e: - assert "object is not iterable" in str(e) + assert "Path or files are required" in str(e) info = await async_sandbox.files.write(filename, content) assert info.path == f"/home/user/{filename}" @@ -30,13 +30,13 @@ async def test_write_multiple_files(async_sandbox: AsyncSandbox): try: await async_sandbox.files.write(None, []) except Exception as e: - assert "object is not iterable" in str(e) + assert "Path or files are required" in str(e) # Attempt to write with path and file array try: await async_sandbox.files.write("/path/to/file", [{ "path": "one_test_file.txt", "data": "This is a test file." }]) except Exception as e: - assert "Cannot specify path with array of files" in str(e) + assert "Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files." in str(e) # Attempt to write with one file in array info = await async_sandbox.files.write([{ "path": "one_test_file.txt", "data": "This is a test file." }]) diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py index 0a97b542b..5a3638765 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_write.py @@ -8,7 +8,7 @@ def test_write_file(sandbox): try: sandbox.files.write(None, content) except Exception as e: - assert "object is not iterable" in str(e) + assert "Path or files are required" in str(e) info = sandbox.files.write(filename, content) assert info.path == f"/home/user/{filename}" @@ -29,13 +29,13 @@ def test_write_multiple_files(sandbox): try: sandbox.files.write(None, []) except Exception as e: - assert "object is not iterable" in str(e) + assert "Path or files are required" in str(e) # Attempt to write with path and file array try: sandbox.files.write("/path/to/file", [{ "path": "one_test_file.txt", "data": "This is a test file." }]) except Exception as e: - assert "Cannot specify path with array of files" in str(e) + assert "Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files." in str(e) # Attempt to write with one file in array info = sandbox.files.write([{ "path": "one_test_file.txt", "data": "This is a test file." }]) From c277f9f8a6727e07c3cc3bf0cbe81c74030fb91f Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 10 Oct 2024 12:23:02 +0200 Subject: [PATCH 16/32] better error messages in js-sdk --- packages/js-sdk/src/sandbox/filesystem/index.ts | 10 +++++++++- packages/js-sdk/tests/sandbox/files/write.test.ts | 9 ++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index 7195b6bca..770d850ae 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -147,8 +147,14 @@ export class Filesystem { dataOrOpts?: WriteData | FilesystemRequestOpts, opts?: FilesystemRequestOpts ): Promise { + if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) { + throw new Error('Path or files are required') + } + if (typeof pathOrFiles === 'string' && Array.isArray(dataOrOpts)) { - throw new Error('Cannot specify path with array of files') + throw new Error( + 'Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files.' + ) } const { path, writeOpts, writeFiles } = @@ -160,6 +166,8 @@ export class Filesystem { } : { path: undefined, writeOpts: dataOrOpts as FilesystemRequestOpts, writeFiles: pathOrFiles as WriteEntry[] } + if (writeFiles.length === 0) return [] as EntryInfo[] + const blobs = await Promise.all(writeFiles.map((f) => new Response(f.data).blob())) const res = await this.envdApi.api.POST('/files', { diff --git a/packages/js-sdk/tests/sandbox/files/write.test.ts b/packages/js-sdk/tests/sandbox/files/write.test.ts index 51a7b968b..b2baa3b44 100644 --- a/packages/js-sdk/tests/sandbox/files/write.test.ts +++ b/packages/js-sdk/tests/sandbox/files/write.test.ts @@ -17,7 +17,7 @@ sandboxTest('write file', async ({ sandbox }) => { }) .catch((err) => { assert.instanceOf(err, Error) - assert.include(err.message, 'Cannot read properties of undefined') + assert.include(err.message, 'Path or files are required') }) const info = await sandbox.files.write(filename, content) @@ -47,7 +47,7 @@ sandboxTest('write multiple files', async ({ sandbox }) => { }) .catch((err) => { assert.instanceOf(err, Error) - assert.include(err.message, 'Cannot read properties of undefined') + assert.include(err.message, 'Path or files are required') }) // Attempt to write with path and file array @@ -59,7 +59,10 @@ sandboxTest('write multiple files', async ({ sandbox }) => { }) .catch((err) => { assert.instanceOf(err, Error) - assert.include(err.message, 'Cannot specify path with array of files') + assert.include( + err.message, + 'Cannot specify both path and array of files. You have to specify either path and data for a single file or an array for multiple files.' + ) }) // Attempt to write with one file in array From 86262f1bc62183b0ac969290463b1e5cc06587e6 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 10 Oct 2024 13:52:53 +0200 Subject: [PATCH 17/32] docstring for dataclass --- packages/python-sdk/e2b/sandbox/filesystem/filesystem.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py index a1dd55210..3eb575791 100644 --- a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py @@ -1,5 +1,5 @@ -from enum import Enum from dataclasses import dataclass +from enum import Enum from typing import IO, Optional, Union from e2b.envd.filesystem import filesystem_pb2 @@ -31,9 +31,15 @@ class EntryInfo: type: Optional[FileType] path: str + WriteData = Union[str, bytes, IO] + @dataclass class WriteEntry: + """ + Contains path and data of the file to be written to the filesystem. + """ + path: str data: WriteData From cf262aeb89c34ce798fcda8e2543d6b0d802c617 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 14:55:17 -0800 Subject: [PATCH 18/32] fix errors.ts comment --- packages/js-sdk/src/errors.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 810888b9e..c4d472f5b 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -6,8 +6,6 @@ export function formatSandboxTimeoutError(message: string) { } /** - * Thrown when a sandbox error occurs. - * * Base class for all sandbox errors. * * Thrown when general sandbox errors occur. From 4ca24147f609db0391fdd76e1c38dfab32fce586 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 15:18:06 -0800 Subject: [PATCH 19/32] fixed typing syntax and watch tests --- .../e2b/sandbox_async/filesystem/filesystem.py | 10 +++++----- .../e2b/sandbox_sync/filesystem/filesystem.py | 10 +++++----- .../tests/sync/sandbox_sync/files/test_watch.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index b55849781..de1b7b4bd 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,5 +1,5 @@ from io import TextIOBase -from typing import AsyncIterator, List, Literal, Optional, overload +from typing import AsyncIterator, List, Literal, Optional, overload, Union from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry import e2b_connect as connect @@ -173,11 +173,11 @@ async def write( async def write( self, - path_or_files: str | List[WriteEntry], - data_or_user: WriteData | Username = "user", - user_or_request_timeout: Optional[float | Username] = None, + path_or_files: Union[str, List[WriteEntry]], + data_or_user: Union[WriteData, Username] = "user", + user_or_request_timeout: Optional[Union[float, Username]] = None, request_timeout_or_none: Optional[float] = None - ) -> EntryInfo | List[EntryInfo]: + ) -> Union[EntryInfo, List[EntryInfo]]: """ Writes content to a file on the path. When writing to a file that doesn't exist, the file will get created. diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 915e3f4d3..811c72384 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,5 +1,5 @@ from io import TextIOBase -from typing import Iterator, List, Literal, Optional, overload +from typing import Iterator, List, Literal, Optional, overload, Union from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry import e2b_connect @@ -173,11 +173,11 @@ def write( def write( self, - path_or_files: str | List[WriteEntry], - data_or_user: WriteData | Username = "user", - user_or_request_timeout: Optional[float | Username] = None, + path_or_files: Union[str, List[WriteEntry]], + data_or_user: Union[WriteData, Username] = "user", + user_or_request_timeout: Optional[Union[float, Username]] = None, request_timeout_or_none: Optional[float] = None - ) -> EntryInfo | List[EntryInfo]: + ) -> Union[EntryInfo, List[EntryInfo]]: path, write_files, user, request_timeout = None, [], "user", None if isinstance(path_or_files, str): if isinstance(data_or_user, list): diff --git a/packages/python-sdk/tests/sync/sandbox_sync/files/test_watch.py b/packages/python-sdk/tests/sync/sandbox_sync/files/test_watch.py index e9fc5faf1..bc4b0afa8 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/files/test_watch.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/files/test_watch.py @@ -13,8 +13,8 @@ def test_watch_directory_changes(sandbox: Sandbox): sandbox.files.write(f"{dirname}/{filename}", content) events = handle.get_new_events() - assert len(events) == 3 - assert events[0].type == FilesystemEventType.CREATE + assert len(events) >= 3 + assert events[0].type == FilesystemEventType.WRITE assert events[0].name == filename assert events[1].type == FilesystemEventType.CHMOD assert events[1].name == filename From 93dc1f9d25bdfaceb36b43325b316f8a2daa8848 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 15:21:16 -0800 Subject: [PATCH 20/32] update docs --- .../(docs)/docs/filesystem/read-write/page.mdx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx index c61c586c0..b66e76ecc 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx @@ -26,12 +26,28 @@ You can write files to the sandbox filesystem using the `files.write()` method. ```js import { Sandbox } from '@e2b/code-interpreter' const sandbox = await Sandbox.create() + +// Write single file await sandbox.files.write('/path/to/file', 'file content') + +// Write multiple files +await sandbox.files.write([ + { path: "/path/to/a", data: "file content" }, + { path: "/another/path/to/b", data: "file content" } +]) ``` ```python from e2b_code_interpreter import Sandbox sandbox = Sandbox() + +# Write single file await sandbox.files.write('/path/to/file', 'file content') + +# Write multiple files +await sandbox.files.write([ + { "path": "/path/to/a", "data": "file content" }, + { "path": "another/path/to/b", "data": "file content" } +]) ``` From 741b32935a5fe8c726aca1c90cc5e446bc3f3544 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 15:25:07 -0800 Subject: [PATCH 21/32] add minor changeset --- .changeset/new-berries-heal.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/new-berries-heal.md diff --git a/.changeset/new-berries-heal.md b/.changeset/new-berries-heal.md new file mode 100644 index 000000000..bf9e0601a --- /dev/null +++ b/.changeset/new-berries-heal.md @@ -0,0 +1,6 @@ +--- +'@e2b/python-sdk': minor +'e2b': minor +--- + +the Filesytem write method is overloaded to also allow passing in an array of files From bbeeb044ccc96e0ed45003ccb7926a5d43ed7ef0 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 15:58:15 -0800 Subject: [PATCH 22/32] upadte upload docs and improve read-write-docs --- .../docs/filesystem/read-write/page.mdx | 32 ++++--- .../(docs)/docs/filesystem/upload/page.mdx | 85 +++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx index b66e76ecc..a16b35a3a 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx @@ -2,7 +2,7 @@ ## Reading files -You can read files from the sandbox filesystem using the `files.reado()` method. +You can read files from the sandbox filesystem using the `files.read()` method. ```js @@ -18,19 +18,35 @@ file_content = sandbox.files.read('/path/to/file') ``` -## Writing files +## Writing single files -You can write files to the sandbox filesystem using the `files.write()` method. +You can write signle files to the sandbox filesystem using the `files.write()` method. ```js import { Sandbox } from '@e2b/code-interpreter' const sandbox = await Sandbox.create() -// Write single file await sandbox.files.write('/path/to/file', 'file content') +``` +```python +from e2b_code_interpreter import Sandbox + +sandbox = Sandbox() + +await sandbox.files.write('/path/to/file', 'file content') +``` + + +## Writing multiple files + +You can also write multiple files to the sandbox filesystem using the `files.write()` method. + + +```js +import { Sandbox } from '@e2b/code-interpreter' +const sandbox = await Sandbox.create() -// Write multiple files await sandbox.files.write([ { path: "/path/to/a", data: "file content" }, { path: "/another/path/to/b", data: "file content" } @@ -41,13 +57,9 @@ from e2b_code_interpreter import Sandbox sandbox = Sandbox() -# Write single file -await sandbox.files.write('/path/to/file', 'file content') - -# Write multiple files await sandbox.files.write([ { "path": "/path/to/a", "data": "file content" }, { "path": "another/path/to/b", "data": "file content" } ]) ``` - + \ No newline at end of file diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index 1b16f2668..79064be71 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -2,6 +2,8 @@ You can upload data to the sandbox using the `files.write()` method. +## Upload single file + ```js import fs from 'fs' @@ -25,3 +27,86 @@ with open("path/to/local/file", "rb") as file: sandbox.files.write("/path/in/sandbox", file) ``` + +## Upload directory / multiple files + + +```js +const fs = require('fs'); +const path = require('path'); + +import { Sandbox } from '@e2b/code-interpreter' + +const sandbox = await Sandbox.create() + +// Read all files in the directory and store their paths and contents in an array +const readDirectoryFiles = (directoryPath) => { + // Read all files in the local directory + const files = fs.readdirSync(directoryPath); + + // Map files to objects with path and data + const filesArray = files + .filter(file => { + const fullPath = path.join(directoryPath, file); + // Skip if it's a directory + return fs.statSync(fullPath).isFile(); + }) + .map(file => { + const filePath = path.join(directoryPath, file); + + // Read the content of each file + return { + path: filePath, + data: fs.readFileSync(filePath, 'utf8') + }; + }); + + return filesArray; +}; + +// Usage example +const files = readDirectoryContents('/local/dir'); +console.log(files); +// [ +// { path: '/local/dir/file1.txt', data: 'File 1 contents...' }, +// { path: '/local/dir/file2.txt', data: 'File 2 contents...' }, +// ... +// ] + +await sandbox.files.write(files) +``` +```python +import os +from e2b_code_interpreter import Sandbox + +sandbox = Sandbox() + +def read_directory_files(directory_path): + files = [] + + # Iterate through all files in the directory + for filename in os.listdir(directory_path): + file_path = os.path.join(directory_path, filename) + + # Skip if it's a directory + if os.path.isfile(file_path): + # Read file contents in binary mode + with open(file_path, "rb") as file: + files.append({ + 'path': file_path, + 'data': file.read() + }) + + return files + +files = read_directory_files('/local/dir'); +print(files); +// [ +// { 'path': '/local/dir/file1.txt', 'data': 'File 1 contents...' }, +// { 'path': '/local/dir/file2.txt', 'data': 'File 2 contents...' }, +// ... +// ] + +sandbox.files.write(files) +``` + \ No newline at end of file From 0e2a684a2cd6a623295017cdc5c43020861b09f7 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 16:06:10 -0800 Subject: [PATCH 23/32] fix indentation in upload docs --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index 79064be71..c58a3847d 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -24,7 +24,7 @@ sandbox = Sandbox() # Read file from local filesystem with open("path/to/local/file", "rb") as file: # Upload file to sandbox - sandbox.files.write("/path/in/sandbox", file) + sandbox.files.write("/path/in/sandbox", file) ``` From b867fe4d6a0f75465c16f88cb0d2b0580f6b86ec Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:07:53 -0800 Subject: [PATCH 24/32] Update apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx index a16b35a3a..c01ad6191 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx @@ -48,7 +48,7 @@ import { Sandbox } from '@e2b/code-interpreter' const sandbox = await Sandbox.create() await sandbox.files.write([ - { path: "/path/to/a", data: "file content" }, + { path: '/path/to/a', data: 'file content' }, { path: "/another/path/to/b", data: "file content" } ]) ``` From e730810fddf00200fdf86ad6e4a277c8a9d8a2c1 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:08:14 -0800 Subject: [PATCH 25/32] Update apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx index c01ad6191..5f17a03cf 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/read-write/page.mdx @@ -49,7 +49,7 @@ const sandbox = await Sandbox.create() await sandbox.files.write([ { path: '/path/to/a', data: 'file content' }, - { path: "/another/path/to/b", data: "file content" } + { path: '/another/path/to/b', data: 'file content' } ]) ``` ```python From d3b99fe7a635531e4a59b569070da0e30b0d9803 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:09:53 -0800 Subject: [PATCH 26/32] Update apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index c58a3847d..e5d572bbe 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -99,7 +99,7 @@ def read_directory_files(directory_path): return files -files = read_directory_files('/local/dir'); +files = read_directory_files("/local/dir") print(files); // [ // { 'path': '/local/dir/file1.txt', 'data': 'File 1 contents...' }, From 424baec371a2dfda12817d022f8608ddae815cc4 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:10:02 -0800 Subject: [PATCH 27/32] Update apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index e5d572bbe..3e69796bc 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -100,7 +100,7 @@ def read_directory_files(directory_path): return files files = read_directory_files("/local/dir") -print(files); +print(files) // [ // { 'path': '/local/dir/file1.txt', 'data': 'File 1 contents...' }, // { 'path': '/local/dir/file2.txt', 'data': 'File 2 contents...' }, From e73ad23bb37110dfc8b4dbf824d4b5b7f4429b63 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:10:11 -0800 Subject: [PATCH 28/32] Update apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index 3e69796bc..fa86a88cd 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -101,7 +101,7 @@ def read_directory_files(directory_path): files = read_directory_files("/local/dir") print(files) -// [ +# [ // { 'path': '/local/dir/file1.txt', 'data': 'File 1 contents...' }, // { 'path': '/local/dir/file2.txt', 'data': 'File 2 contents...' }, // ... From d0fc09a0d4de7b656007cac2d5b2c1fcb69e9fc2 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:10:21 -0800 Subject: [PATCH 29/32] Update apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index fa86a88cd..406904183 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -102,7 +102,7 @@ def read_directory_files(directory_path): files = read_directory_files("/local/dir") print(files) # [ -// { 'path': '/local/dir/file1.txt', 'data': 'File 1 contents...' }, +# {"'path": "/local/dir/file1.txt", "data": "File 1 contents..." }, // { 'path': '/local/dir/file2.txt', 'data': 'File 2 contents...' }, // ... // ] From 1708cc06098572e5e868d7d5f7b05f379d8b2155 Mon Sep 17 00:00:00 2001 From: 0div <98087403+0div@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:11:52 -0800 Subject: [PATCH 30/32] Update apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx Co-authored-by: Vasek Mlejnsky --- apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx index 406904183..b0d7877d7 100644 --- a/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx +++ b/apps/web/src/app/(docs)/docs/filesystem/upload/page.mdx @@ -103,9 +103,9 @@ files = read_directory_files("/local/dir") print(files) # [ # {"'path": "/local/dir/file1.txt", "data": "File 1 contents..." }, -// { 'path': '/local/dir/file2.txt', 'data': 'File 2 contents...' }, -// ... -// ] +# { "path": "/local/dir/file2.txt", "data": "File 2 contents..." }, +# ... +# ] sandbox.files.write(files) ``` From 502c414ce42d581fe9d2c1de7df811065cbb3eb1 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 17:08:16 -0800 Subject: [PATCH 31/32] remove WriteData type in js-sdk --- packages/js-sdk/src/index.ts | 4 +--- packages/js-sdk/src/sandbox/filesystem/index.ts | 15 ++++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index 05f304c0e..17e6e59d8 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -15,7 +15,7 @@ export { export type { Logger } from './logs' export { FileType } from './sandbox/filesystem' -export type { EntryInfo, Filesystem, WriteData } from './sandbox/filesystem' +export type { EntryInfo, Filesystem } from './sandbox/filesystem' export { FilesystemEventType } from './sandbox/filesystem/watchHandle' export type { FilesystemEvent, WatchHandle } from './sandbox/filesystem/watchHandle' @@ -38,8 +38,6 @@ export type { Pty, } from './sandbox/commands' -export type { Pty } from './sandbox/pty' - export type { SandboxOpts } from './sandbox' export type { SandboxInfo } from './sandbox/sandboxApi' export { Sandbox } diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts index c330b88cc..68f8c3c4b 100644 --- a/packages/js-sdk/src/sandbox/filesystem/index.ts +++ b/packages/js-sdk/src/sandbox/filesystem/index.ts @@ -54,11 +54,9 @@ export const enum FileType { DIR = 'dir', } -export type WriteData = string | ArrayBuffer | Blob | ReadableStream - export type WriteEntry = { path: string - data: WriteData + data: string | ArrayBuffer | Blob | ReadableStream } function mapFileType(fileType: FsFileType) { @@ -227,11 +225,11 @@ export class Filesystem { * * @returns information about the written file */ - async write(path: string, data: WriteData, opts?: FilesystemRequestOpts): Promise + async write(path: string, data: string | ArrayBuffer | Blob | ReadableStream, opts?: FilesystemRequestOpts): Promise async write(files: WriteEntry[], opts?: FilesystemRequestOpts): Promise async write( pathOrFiles: string | WriteEntry[], - dataOrOpts?: WriteData | FilesystemRequestOpts, + dataOrOpts?: string | ArrayBuffer | Blob | ReadableStream | FilesystemRequestOpts, opts?: FilesystemRequestOpts ): Promise { if (typeof pathOrFiles !== 'string' && !Array.isArray(pathOrFiles)) { @@ -249,7 +247,7 @@ export class Filesystem { ? { path: pathOrFiles, writeOpts: opts as FilesystemRequestOpts, - writeFiles: [{ data: dataOrOpts as WriteData }], + writeFiles: [{ data: dataOrOpts as string | ArrayBuffer | Blob | ReadableStream }], } : { path: undefined, writeOpts: dataOrOpts as FilesystemRequestOpts, writeFiles: pathOrFiles as WriteEntry[] } @@ -462,7 +460,10 @@ export class Filesystem { async watchDir( path: string, onEvent: (event: FilesystemEvent) => void | Promise, - opts?: WatchOpts + opts?: FilesystemRequestOpts & { + timeout?: number + onExit?: (err?: Error) => void | Promise + } ): Promise { const requestTimeoutMs = opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs From 724fbc63cc13e79b6acf9a3e666bc2d79ae50ba0 Mon Sep 17 00:00:00 2001 From: 0div Date: Thu, 12 Dec 2024 17:13:09 -0800 Subject: [PATCH 32/32] remove WriteData type in python-sdk --- packages/python-sdk/e2b/sandbox/filesystem/filesystem.py | 7 ++----- .../python-sdk/e2b/sandbox_async/filesystem/filesystem.py | 8 ++++---- .../python-sdk/e2b/sandbox_sync/filesystem/filesystem.py | 8 ++++---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py index 9ccc017f9..2f6717014 100644 --- a/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox/filesystem/filesystem.py @@ -47,14 +47,11 @@ class EntryInfo: """ -WriteData = Union[str, bytes, IO] - - -@dataclass +dataclass class WriteEntry: """ Contains path and data of the file to be written to the filesystem. """ path: str - data: WriteData + data: Union[str, bytes, IO] diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py index de1b7b4bd..adef7a66f 100644 --- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py @@ -1,6 +1,6 @@ from io import TextIOBase -from typing import AsyncIterator, List, Literal, Optional, overload, Union -from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry +from typing import AsyncIterator, IO, List, Literal, Optional, overload, Union +from e2b.sandbox.filesystem.filesystem import WriteEntry import e2b_connect as connect import httpcore @@ -134,7 +134,7 @@ async def read( async def write( self, path: str, - data: WriteData, + data: Union[str, bytes, IO], user: Username = "user", request_timeout: Optional[float] = None, ) -> EntryInfo: @@ -174,7 +174,7 @@ async def write( async def write( self, path_or_files: Union[str, List[WriteEntry]], - data_or_user: Union[WriteData, Username] = "user", + data_or_user: Union[str, bytes, IO, Username] = "user", user_or_request_timeout: Optional[Union[float, Username]] = None, request_timeout_or_none: Optional[float] = None ) -> Union[EntryInfo, List[EntryInfo]]: diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py index 811c72384..5c73111fa 100644 --- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py +++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py @@ -1,6 +1,6 @@ from io import TextIOBase -from typing import Iterator, List, Literal, Optional, overload, Union -from e2b.sandbox.filesystem.filesystem import WriteData, WriteEntry +from typing import IO, Iterator, List, Literal, Optional, overload, Union +from e2b.sandbox.filesystem.filesystem import WriteEntry import e2b_connect import httpcore @@ -131,7 +131,7 @@ def read( def write( self, path: str, - data: WriteData, + data: Union[str, bytes, IO], user: Username = "user", request_timeout: Optional[float] = None, ) -> EntryInfo: @@ -174,7 +174,7 @@ def write( def write( self, path_or_files: Union[str, List[WriteEntry]], - data_or_user: Union[WriteData, Username] = "user", + data_or_user: Union[str, bytes, IO, Username] = "user", user_or_request_timeout: Optional[Union[float, Username]] = None, request_timeout_or_none: Optional[float] = None ) -> Union[EntryInfo, List[EntryInfo]]: