Skip to content

Commit

Permalink
wip repos
Browse files Browse the repository at this point in the history
  • Loading branch information
jonathanfallon committed Jan 19, 2024
1 parent c85df9f commit 599d0f6
Show file tree
Hide file tree
Showing 8 changed files with 307 additions and 49 deletions.
13 changes: 13 additions & 0 deletions api/services/export/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,16 @@ En cas d'erreur :

- notifier l'utilisateur
- notifier l'administration technique du site

# TODO

- [ ] gérer les champs en fonction du type d'export (config)
- [ ] tester les repositories
- [ ] migrations
- [ ] tests
- [ ] models (export, recipient, log)
- [ ] validation (schemas)
- [ ] permissions
- [ ] actions
- [ ] commands
- [ ] CRON jobs (infra)
5 changes: 4 additions & 1 deletion api/services/export/src/ServiceProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ import { DebugCommand } from './commands/DebugCommand';
import { config } from './config';
import { CampaignRepository } from './repositories/CampaignRepository';
import { CarpoolRepository } from './repositories/CarpoolRepository';
import { LogRepository } from './repositories/LogRepository';
import { BuildService } from './services/BuildService';
import { RecipientRepository } from './repositories/RecipientRepository';
import { ExportRepository } from './repositories/ExportRepository';

// Services are from the ./services folder
// and are used to implement the business logic of the application.
Expand All @@ -19,7 +22,7 @@ const services = [BuildService];

// Repositories are from the ./repositories folder
// and are used to access the database or other data sources.
const repositories = [CarpoolRepository, CampaignRepository];
const repositories = [ExportRepository, RecipientRepository, CampaignRepository, CarpoolRepository, LogRepository];

// External providers are from the @pdc namespace
const externalProviders = [S3StorageProvider];
Expand Down
73 changes: 29 additions & 44 deletions api/services/export/src/commands/DebugCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,53 +15,38 @@ export class DebugCommand implements CommandInterface {

public async call(): Promise<void> {
// TODO normalise params
const params = new ExportParams({
start_at: new Date('2023-08-14T00:00:00+0200'),
end_at: new Date('2023-08-15T00:00:00+0200'),
});

// TODO create ExportFile entity and pass it to the provider
// TODO get the file name from the config as done in APDFNameProvider
const fileWriter = new XLSXWriter('send-test', {
compress: true,
fields: ['trip_id', 'policy_id', 'operator', 'campaign_mode'],

// define computed fields to be added to the carpool row
// it takes a name (snake case, lowercase, no spaces, no special characters...)
// and a compute function which will bind the datasources together to get the result.
//
// Try to put the logic in the models and use the compute function as a controller to
// bind tools together.
//
// Keep in mind this will run for EVERY SINGLE row of the file, so keep it fast
// and preload data in the datasources in the calling service.
computed: [
{
name: 'campaign_mode',
compute(row, datasources) {
const campaign = datasources.get('campaigns').get(row.value('campaign_id'));
return campaign && campaign.getModeAt([row.value('start_datetime_utc'), row.value('end_datetime_utc')]);
},
},
],
});

// create the Workbook and write data
try {
await fileWriter.create();
await this.build.run(params, fileWriter);
await fileWriter.printHelp();
} catch (e) {
console.error(e.message);
} finally {
await fileWriter.close();
console.info(`File written to ${fileWriter.workbookPath}`);
// TODO cleanup on failure
}

// compress the file
await fileWriter.compress();
console.info(`File compressed to ${fileWriter.archivePath}`);
await this.build.run(
new ExportParams({
start_at: new Date('2023-08-14T00:00:00+0200'),
end_at: new Date('2023-08-15T00:00:00+0200'),
}),
new XLSXWriter('send-test', {
compress: true,
fields: ['trip_id', 'policy_id', 'operator', 'campaign_mode'],

// define computed fields to be added to the carpool row
// it takes a name (snake case, lowercase, no spaces, no special characters...)
// and a compute function which will bind the datasources together to get the result.
//
// Try to put the logic in the models and use the compute function as a controller to
// bind tools together.
//
// Keep in mind this will run for EVERY SINGLE row of the file, so keep it fast
// and preload data in the datasources in the calling service.
computed: [
{
name: 'campaign_mode',
compute(row, datasources) {
const campaign = datasources.get('campaigns').get(row.value('campaign_id'));
return campaign && campaign.getModeAt([row.value('start_datetime_utc'), row.value('end_datetime_utc')]);
},
},
],
}),
);

// TODO upload
// TODO cleanup
Expand Down
6 changes: 5 additions & 1 deletion api/services/export/src/models/XLSXWriter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,11 @@ export class XLSXWriter {

// TODO compress the file with ZIP (for now)
public async compress(): Promise<XLSXWriter> {
// TODO
if (!this.options.compress) {
console.info(`Skipped compression of ${this.workbookPath}`);
return this;
}

const zip = new AdmZip();
zip.addLocalFile(this.workbookPath);
zip.writeZip(this.archivePath);
Expand Down
107 changes: 107 additions & 0 deletions api/services/export/src/repositories/ExportRepository.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import { provider } from '@ilos/common';
import { PostgresConnection } from '@ilos/connection-postgres';
import { ExportParams } from '../models/ExportParams';

export type Export = {
_id: number;
created_at: Date;
updated_at: Date;
created_by: number;
uuid: string;
status: ExportStatus;
progress: number;
type: ExportType;
download_url: string;
params: ExportParams;
error: any;
stats: any;
};
export enum ExportStatus {
PENDING = 'pending',
RUNNING = 'running',
SUCCESS = 'success',
FAILURE = 'failure',
}
export enum ExportType {
REGULAR = 'regular',
OPENDATA = 'opendata',
OPERATOR = 'operator',
TERRITORY = 'territory',
REGISTRY = 'registry',
}
export type ExportCreateData = Pick<Export, 'created_by' | 'type' | 'params'>;
export type ExportUpdateData = Partial<Pick<Export, 'status' | 'progress' | 'download_url' | 'error' | 'stats'>>;

export interface ExportRepositoryInterface {
create(data: ExportCreateData): Promise<number>;
get(id: number): Promise<any>;
update(id: number, data: ExportUpdateData): Promise<void>;
delete(id: number): Promise<void>;
list(): Promise<any[]>;
}

export abstract class ExportRepositoryInterfaceResolver implements ExportRepositoryInterface {
public async create(data: ExportCreateData): Promise<number> {
throw new Error('Not implemented');
}
public async get(id: number): Promise<any> {
throw new Error('Not implemented');
}
public async update(id: number, data: ExportUpdateData): Promise<void> {
throw new Error('Not implemented');
}
public async delete(id: number): Promise<void> {
throw new Error('Not implemented');
}
public async list(): Promise<any[]> {
throw new Error('Not implemented');
}
}

@provider({
identifier: ExportRepositoryInterfaceResolver,
})
export class ExportRepository implements ExportRepositoryInterface {
protected readonly table = 'export.exports';

constructor(protected connection: PostgresConnection) {}

public async create(data: ExportCreateData): Promise<number> {
const { rows } = await this.connection.getClient().query({
text: `INSERT INTO ${this.table} (created_by, type, params) VALUES ($1, $2, $3) RETURNING _id`,
values: [data.created_by, data.type, data.params],
});
return rows[0]._id;
}

public async get(id: number): Promise<any> {
const { rows } = await this.connection.getClient().query({
text: `SELECT * FROM ${this.table} WHERE _id = $1`,
values: [id],
});
return rows[0];
}

public async update(id: number, data: ExportUpdateData): Promise<void> {
await this.connection.getClient().query({
text: `UPDATE ${this.table} SET ${Object.keys(data)
.map((key, index) => `${key} = $${index + 2}`)
.join(', ')} WHERE _id = $1`,
values: [id, ...Object.values(data)],
});
}

public async delete(id: number): Promise<void> {
await this.connection.getClient().query({
text: `DELETE FROM ${this.table} WHERE _id = $1`,
values: [id],
});
}

public async list(): Promise<any[]> {
const { rows } = await this.connection.getClient().query({
text: `SELECT * FROM ${this.table}`,
});
return rows;
}
}
33 changes: 33 additions & 0 deletions api/services/export/src/repositories/LogRepository.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { provider } from '@ilos/common';
import { PostgresConnection } from '@ilos/connection-postgres';

export enum LogStatus {
SUCCESS = 'success',
FAILURE = 'failure',
}

export interface LogRepositoryInterface {
add(export_id: number, type: LogStatus, message: string): Promise<void>;
}

export abstract class LogRepositoryInterfaceResolver implements LogRepositoryInterface {
public async add(export_id: number, type: LogStatus, message: string): Promise<void> {
throw new Error('Not implemented');
}
}

@provider({
identifier: LogRepositoryInterfaceResolver,
})
export class LogRepository implements LogRepositoryInterface {
protected readonly table = 'export.logs';

constructor(protected connection: PostgresConnection) {}

public async add(export_id: number, type: LogStatus, message: string): Promise<void> {
await this.connection.getClient().query({
text: `INSERT INTO ${this.table} (export_id, type, message) VALUES ($1, $2, $3)`,
values: [export_id, type, message],
});
}
}
62 changes: 62 additions & 0 deletions api/services/export/src/repositories/RecipientRepository.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { provider } from '@ilos/common';
import { PostgresConnection } from '@ilos/connection-postgres';

export type Recipient = {
_id: number;
scrambled_at: Date;
export_id: number;
email: string;
fullname: string;
message: string;
};

export type CreateRecipientData = Pick<Recipient, 'export_id' | 'email' | 'fullname' | 'message'>;

export interface RecipientRepositoryInterface {
create(data: CreateRecipientData): Promise<number>;
anonymize(export_id: number): Promise<void>;
}

export abstract class RecipientRepositoryInterfaceResolver implements RecipientRepositoryInterface {
public async create(data: CreateRecipientData): Promise<number> {
throw new Error('Not implemented');
}
public async anonymize(export_id: number): Promise<void> {
throw new Error('Not implemented');
}
}

@provider({
identifier: RecipientRepositoryInterfaceResolver,
})
export class RecipientRepository implements RecipientRepositoryInterface {
protected readonly table = 'export.recipients';

constructor(protected connection: PostgresConnection) {}

public async create(data: CreateRecipientData): Promise<number> {
const { rows } = await this.connection.getClient().query({
text: `
INSERT INTO ${this.table}
(export_id, email, fullname, message)
VALUES ($1, $2, $3, $4)
RETURNING _id`,
values: [data.export_id, data.email, data.fullname, data.message],
});
return rows[0]._id;
}

public async anonymize(export_id: number): Promise<void> {
await this.connection.getClient().query({
text: `
UPDATE ${this.table}
SET
scrambled_at = NOW(),
email = NULL,
fullname = NULL,
message = NULL
WHERE export_id = $1`,
values: [export_id],
});
}
}
Loading

0 comments on commit 599d0f6

Please sign in to comment.