Skip to content

Commit

Permalink
Feature: Make File Public (#25)
Browse files Browse the repository at this point in the history
  • Loading branch information
@perfectmak
perfectmak authored Feb 8, 2021
1 parent a760c7f commit bbd4627
Show file tree
Hide file tree
Showing 8 changed files with 247 additions and 49 deletions.
19 changes: 16 additions & 3 deletions etc/sdk.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,8 @@ export interface FileMember {

// @public
export interface FileMetadata {
// (undocumented)
bucketKey?: string;
// (undocumented)
bucketSlug: string;
// (undocumented)
Expand Down Expand Up @@ -178,10 +180,10 @@ export class GundbMetadataStore implements UserMetadataStore {
findBucket(bucketSlug: string): Promise<BucketMetadata | undefined>;
findFileMetadata(bucketSlug: string, dbId: string, path: string): Promise<FileMetadata | undefined>;
findFileMetadataByUuid(uuid: string): Promise<FileMetadata | undefined>;
// Warning: (ae-forgotten-export) The symbol "GunChainReference" needs to be exported by the entry point index.d.ts
// Warning: (ae-forgotten-export) The symbol "GunDataState" needs to be exported by the entry point index.d.ts
static fromIdentity(username: string, userpass: string, gunOrServer?: GunChainReference<GunDataState> | string, logger?: Pino.Logger | boolean): Promise<GundbMetadataStore>;
// Warning: (ae-forgotten-export) The symbol "GunInit" needs to be exported by the entry point index.d.ts
static fromIdentity(username: string, userpass: string, gunOrServer?: GunInit | string | string[], logger?: Pino.Logger | boolean): Promise<GundbMetadataStore>;
listBuckets(): Promise<BucketMetadata[]>;
setFilePublic(metadata: FileMetadata): Promise<void>;
upsertFileMetadata(metadata: FileMetadata): Promise<FileMetadata>;
}

Expand Down Expand Up @@ -225,6 +227,15 @@ export interface ListDirectoryResponse {
items: DirectoryEntry[];
}

// @public (undocumented)
export interface MakeFilePublicRequest {
allowAccess: boolean;
// (undocumented)
bucket: string;
// (undocumented)
path: string;
}

// @public (undocumented)
export interface OpenFileRequest {
// (undocumented)
Expand Down Expand Up @@ -336,6 +347,7 @@ export interface UserMetadataStore {
findFileMetadata: (bucketSlug: string, dbId: string, path: string) => Promise<FileMetadata | undefined>;
findFileMetadataByUuid: (uuid: string) => Promise<FileMetadata | undefined>;
listBuckets: () => Promise<BucketMetadata[]>;
setFilePublic: (metadata: FileMetadata) => Promise<void>;
upsertFileMetadata: (data: FileMetadata) => Promise<FileMetadata>;
}

Expand Down Expand Up @@ -371,6 +383,7 @@ export class UserStorage {
listDirectory(request: ListDirectoryRequest): Promise<ListDirectoryResponse>;
openFile(request: OpenFileRequest): Promise<OpenFileResponse>;
openFileByUuid(request: OpenUuidFileRequest): Promise<OpenUuidFileResponse>;
setFilePublicAccess(request: MakeFilePublicRequest): Promise<void>;
txlSubscribe(): Promise<TxlSubscribeResponse>;
}

Expand Down
12 changes: 11 additions & 1 deletion integration_tests/storage_interactions.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ describe('Users storing data', () => {
expect(fileResponse?.entry?.bucket).to.not.be.empty;
expect(fileResponse?.entry?.dbId).to.not.be.empty;
expect(fileResponse.entry.name).to.equal('top.txt');
const actualTxtContent = await fileResponse.consumeStream();
let actualTxtContent = await fileResponse.consumeStream();
expect(new TextDecoder('utf8').decode(actualTxtContent)).to.equal(txtContent);

// ensure file is not accessible from outside of owners file
Expand All @@ -231,6 +231,16 @@ describe('Users storing data', () => {

await expect(unauthorizedStorage.openFileByUuid({ uuid: file?.uuid || '' }))
.to.eventually.be.rejectedWith(FileNotFoundError);

// ensure file is accessible after making it public
await storage.setFilePublicAccess({ bucket: 'personal', path: '/top.txt', allowAccess: true });

const publicFileResponse = await unauthorizedStorage.openFileByUuid({
uuid: file?.uuid || '',
});
expect(publicFileResponse.entry.name).to.equal('top.txt');
actualTxtContent = await publicFileResponse.consumeStream();
expect(new TextDecoder('utf8').decode(actualTxtContent)).to.equal(txtContent);
}).timeout(TestsDefaultTimeout);

it('should subscribe to textile events', async () => {
Expand Down
35 changes: 33 additions & 2 deletions packages/storage/src/metadata/gundbMetadataStore.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Identity } from '@spacehq/users';
import { isBrowser } from 'browser-or-node';
import { PrivateKey } from '@textile/crypto';
import * as chaiAsPromised from 'chai-as-promised';
import * as chaiSubset from 'chai-subset';
Expand All @@ -13,6 +14,11 @@ const identity: Identity = PrivateKey.fromRandom();
const username = Buffer.from(identity.public.pubKey).toString('hex');
const password = Buffer.from(identity.privKey).toString('hex');
describe('GunsdbMetadataStore', () => {
if (!isBrowser) {
// TODO: Fix this for node. It fails because node isn't properly authenticating public account with server
return;
}

it('should work', async () => {
const bucket = 'personal';
const dbId = 'something';
Expand All @@ -36,7 +42,7 @@ describe('GunsdbMetadataStore', () => {

const existingBuckets = await newStore.listBuckets();
expect(existingBuckets).to.containSubset([{ dbId, slug: bucket }]);
}).timeout(10000);
}).timeout(90000);

it('should work for file metadata', async () => {
const bucketSlug = 'personal';
Expand All @@ -59,5 +65,30 @@ describe('GunsdbMetadataStore', () => {
// test retrieving by uuid
const savedMetadataByUuid = await store.findFileMetadataByUuid(fileMetadata.uuid);
expect(savedMetadataByUuid).to.deep.equal(fileMetadata);
}).timeout(10000);
}).timeout(90000);

it('should fetch public file metadata correctly', async () => {
const store = await GundbMetadataStore.fromIdentity(username, password, undefined, false);
const fileMetadata = {
uuid: v4(),
mimeType: 'image/png',
bucketSlug: 'personal',
dbId: 'something',
path: '/home/case.png',
};

await store.upsertFileMetadata(fileMetadata);
await store.setFilePublic(fileMetadata);

// test retrieving by uuid from another user
const newUser: Identity = PrivateKey.fromRandom();
const newStore = await GundbMetadataStore.fromIdentity(
Buffer.from(newUser.public.pubKey).toString('hex'),
Buffer.from(newUser.privKey).toString('hex'),
undefined,
false,
);
const savedMetadataByUuid = await newStore.findFileMetadataByUuid(fileMetadata.uuid);
expect(savedMetadataByUuid).to.deep.equal(fileMetadata);
}).timeout(90000);
});
119 changes: 96 additions & 23 deletions packages/storage/src/metadata/gundbMetadataStore.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* eslint-disable no-underscore-dangle */
/* eslint-disable no-underscore-dangle,@typescript-eslint/no-var-requires,global-require */
import { isNode } from 'browser-or-node';
import Pino from 'pino';
import { IGunChainReference } from 'gun/types/chain';
Expand All @@ -7,16 +7,16 @@ import { BucketMetadata, FileMetadata, UserMetadataStore } from './metadataStore

let Gun: IGunStatic;
if (isNode) {
// eslint-disable-next-line @typescript-eslint/no-var-requires,global-require
Gun = require('gun');
} else {
// eslint-disable-next-line @typescript-eslint/no-var-requires,global-require
Gun = require('gun/gun');
// eslint-disable-next-line @typescript-eslint/no-var-requires,global-require
require('gun/sea');
require('gun/lib/radix');
require('gun/lib/radisk');
require('gun/lib/store');
require('gun/lib/rindexed');
}

// eslint-disable-next-line @typescript-eslint/no-var-requires
const crypto = require('crypto-browserify');

// this is an hack to enable using IGunChainReference in async functions
Expand All @@ -25,26 +25,35 @@ export type GunChainReference<Data> = Omit<IGunChainReference<Data>, 'then'>;
// 32 bytes aes key + 16 bytes salt/IV + 32 bytes HMAC key
const BucketEncryptionKeyLength = 32 + 16 + 32;
const BucketMetadataCollection = 'BucketMetadata';
const PublicStoreUsername = '66f47ce32570335085b39bdf';
const PublicStorePassword = '830a20694358651ef14e472fd71c4f9f843ecd50784b241a6c9999dba4c6fced0f90c686bdee28edc';

interface AckError {
err: string;
}

// Remapped bucket metadata type compatible with Gundb
type GunBucketMetadata = Omit<BucketMetadata, 'encryptionKey'> & { encryptionKey: string };
type GunFileMetadata = { data: string };
type EncryptedMetadata = { data: string; };

interface LookupDataState {
[dbIdBucket: string]: EncryptedMetadata;
}

interface LookupFileMetadataState {
[lookupId: string]: GunFileMetadata;
}

interface ListDataState {
[collectionName: string]: EncryptedMetadata[]
}

// Data schema of records stored in gundb
// currently only a single bucket metadata collection
export type GunDataState = LookupDataState | ListDataState;
export type GunDataState = LookupDataState | ListDataState | LookupFileMetadataState;

type GunInit = (() => GunChainReference<GunDataState>);

/**
* A Users Storage Metadata store backed by gundsdb.
Expand All @@ -53,13 +62,15 @@ export type GunDataState = LookupDataState | ListDataState;
*
*/
export class GundbMetadataStore implements UserMetadataStore {
private readonly gun: GunChainReference<GunDataState>;
private gunInit: GunInit;

// in memory cache list of buckets
private bucketsListCache: BucketMetadata[];

private _user?: GunChainReference<GunDataState>;

private _publicUser?: GunChainReference<GunDataState>;

private logger?: Pino.Logger;

/**
Expand All @@ -69,17 +80,21 @@ export class GundbMetadataStore implements UserMetadataStore {
private constructor(
private readonly username: string,
private readonly userpass: string,
gunOrServer?: GunChainReference<GunDataState> | string,
gunOrServer?: GunInit | string | string[],
logger?: Pino.Logger | boolean,
) {
if (gunOrServer) {
if (typeof gunOrServer === 'string') {
this.gun = Gun({ web: gunOrServer });
if (typeof gunOrServer === 'string' || Array.isArray(gunOrServer)) {
this.gunInit = () => Gun(gunOrServer);
} else {
this.gun = gunOrServer;
this.gunInit = gunOrServer;
}
} else {
this.gun = Gun(['https://gun.space.storage/gun']);
this.gunInit = () => Gun({
localStorage: false,
radisk: true,
peers: 'https://gun.space.storage/gun',
} as any);
}

this.bucketsListCache = [];
Expand All @@ -106,11 +121,16 @@ export class GundbMetadataStore implements UserMetadataStore {
static async fromIdentity(
username: string,
userpass: string,
gunOrServer?: GunChainReference<GunDataState> | string,
gunOrServer?: GunInit | string | string[],
logger?: Pino.Logger | boolean,
): Promise<GundbMetadataStore> {
const store = new GundbMetadataStore(username, userpass, gunOrServer, logger);
await store.authenticateUser();

store._user = store.gunInit().user();
await store.authenticateUser(store._user, username, userpass);
store._publicUser = store.gunInit().user();
await store.authenticateUser(store._publicUser, PublicStoreUsername, PublicStorePassword);

await store.startCachingBucketsList();

return store;
Expand Down Expand Up @@ -225,7 +245,28 @@ export class GundbMetadataStore implements UserMetadataStore {
public async findFileMetadataByUuid(uuid: string): Promise<FileMetadata | undefined> {
this.logger?.info({ uuid }, 'Store.findFileMetadataByUuid');
const lookupKey = GundbMetadataStore.getFilesUuidLookupKey(uuid);
return this.lookupFileMetadata(lookupKey);

// NOTE: This can be speedup by making this fetch promise a race instead of sequential
return this.lookupFileMetadata(lookupKey).then((data) => {
if (!data) {
return this.lookupPublicFileMetadata(lookupKey);
}
return data;
});
}

/**
* {@inheritDoc @spacehq/sdk#UserMetadataStore.setFilePublic}
*/
public async setFilePublic(metadata: FileMetadata): Promise<void> {
if (metadata.uuid === undefined) {
throw new Error('metadata file must have a uuid');
}

const lookupKey = GundbMetadataStore.getFilesUuidLookupKey(metadata.uuid);
this.logger?.info({ metadata, lookupKey }, 'Making file metadata public');

this.publicLookupChain.get(lookupKey).put({ data: JSON.stringify(metadata) });
}

private async lookupFileMetadata(lookupKey: string): Promise<FileMetadata | undefined> {
Expand All @@ -239,6 +280,7 @@ export class GundbMetadataStore implements UserMetadataStore {
if (!encryptedData) {
this.logger?.info({ lookupKey }, 'FileMetadata not found');
resolve(undefined);
return;
}

try {
Expand All @@ -252,6 +294,21 @@ export class GundbMetadataStore implements UserMetadataStore {
});
}

private async lookupPublicFileMetadata(lookupKey: string): Promise<FileMetadata | undefined> {
return new Promise<FileMetadata | undefined>((resolve, reject) => {
this.publicLookupChain.get(lookupKey).get('data').once((data) => {
if (!data) {
this.logger?.info({ lookupKey }, 'Public FileMetadata not found');
resolve(undefined);
return;
}

this.logger?.info({ lookupKey }, 'Public FileMetadata found');
resolve(JSON.parse(data));
});
});
}

private async startCachingBucketsList(): Promise<void> {
this.listUser.get(BucketMetadataCollection).map().once(async (data) => {
if (data) {
Expand Down Expand Up @@ -289,6 +346,15 @@ export class GundbMetadataStore implements UserMetadataStore {
return this._user!;
}

private get publicUser(): GunChainReference<GunDataState> {
if (!this._publicUser || !(this._publicUser as unknown as { is?: Record<never, never>; }).is) {
throw new Error('gundb user not authenticated');
}

// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
return this._publicUser!;
}

// use this alias getter for lookuping up users metadata so typescript works as expected
private get lookupUser(): GunChainReference<LookupDataState> {
return this.user as GunChainReference<LookupDataState>;
Expand All @@ -299,25 +365,32 @@ export class GundbMetadataStore implements UserMetadataStore {
return this.user as GunChainReference<ListDataState>;
}

private async authenticateUser(): Promise<void> {
private get publicLookupChain(): GunChainReference<LookupFileMetadataState> {
return this.publicUser as GunChainReference<LookupFileMetadataState>;
}

// eslint-disable-next-line class-methods-use-this
private async authenticateUser<T>(
user: GunChainReference<T>,
username: string,
userpass: string,
): Promise<void> {
this.logger?.info({ username }, 'Authenticating user');
// user.is checks if user is currently logged in
if (this._user && (this._user as unknown as { is?: Record<never, never>; }).is) {
if ((user as unknown as { is?: Record<never, never>; }).is) {
this.logger?.info({ username }, 'User already authenticated');
return;
}

this._user = this.gun.user();

await new Promise((resolve, reject) => {
// eslint-disable-next-line no-unused-expressions
this._user?.create(this.username, this.userpass, (ack) => {
user.create(username, userpass, (ack) => {
// if ((ack as AckError).err) {
// // error here means user either exists or is being created, see gundb user docs.
// // so ignoring
// return;
// }

// eslint-disable-next-line no-unused-expressions
this._user?.auth(this.username, this.userpass, (auth) => {
user.auth(username, userpass, (auth) => {
if ((auth as AckError).err) {
reject(new Error(`gundb failed to authenticate user: ${(auth as AckError).err}`));
return;
Expand Down
Loading

0 comments on commit bbd4627

Please sign in to comment.