Skip to content

Commit

Permalink
explode incentive fields in SQL
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathanfallon
jonathanfallon committed Feb 6, 2024
1 parent e1ae74a commit f638be5
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 86 deletions.
19 changes: 16 additions & 3 deletions api/services/export/src/models/CarpoolRow.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,22 @@ export type CarpoolRowData = {
driver_revenue: number;
passenger_contribution: number;
campaign_id: number;
incentive: Array<Incentive>;
incentive_rpc: Array<IncentiveRPC>;
incentive_counterpart: Array<IncentiveCounterpart>;

incentive_0_index: number;
incentive_0_siret: string;
incentive_0_amount: number;
incentive_1_index: number;
incentive_1_siret: string;
incentive_1_amount: number;
incentive_2_index: number;
incentive_2_siret: string;

incentive_rpc_0_campaign_id: number;
incentive_rpc_0_amount: number;
incentive_rpc_1_campaign_id: number;
incentive_rpc_1_amount: number;
incentive_rpc_2_campaign_id: number;
incentive_rpc_2_amount: number;

offer_public: boolean;
offer_accepted_at: Date;
Expand Down
25 changes: 0 additions & 25 deletions api/services/export/src/models/XLSXWriter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import os from 'node:os';
import path from 'node:path';
import { ExportType } from '../repositories/ExportRepository';
import { AllowedComputedFields, CarpoolRow, CarpoolRowData } from './CarpoolRow';
import { get } from 'lodash';

export type Datasources = Map<string, any>;

Expand Down Expand Up @@ -68,9 +67,6 @@ export class XLSXWriter {
this.options = { ...this.options, ...config } as Options;
this.folder = os.tmpdir();
this.basename = this.sanitize(filename);

// add computed fields and build the list to explide incentive data in many fields
this.options.computed = [...this.options.computed, ...this.computeIncentivesFunctions()];
}

// TODO create the workbook and the worksheets
Expand Down Expand Up @@ -166,25 +162,4 @@ export class XLSXWriter {
.toLowerCase()
.substring(0, 128);
}

protected computeIncentivesFunctions(): ComputedProcessors {
return this.options.fields
.filter((fieldName) => fieldName.startsWith('incentive_'))
.flatMap((fieldName: keyof AllowedComputedFields) => {
const [type, index, ...parts] = fieldName.split('_').reverse();
const dbName = parts.reverse().join('_') as keyof Pick<
CarpoolRowData,
'incentive' | 'incentive_rpc' | 'incentive_counterpart'
>;
return {
name: fieldName,
compute(row): AllowedComputedFields[typeof fieldName] | null {
const src = row.value(dbName);
const idx = parseInt(index);
const val = get(src, `${(isNaN(idx) ? 0 : idx) - 1}.${type}`, null);
return type === 'amount' && val ? parseInt(val) / 100 : val;
},
};
});
}
}
133 changes: 75 additions & 58 deletions api/services/export/src/repositories/queries/CarpoolListQuery.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,56 +8,56 @@ export class CarpoolListQuery extends Query {
cc.acquisition_id,
cc.trip_id,
cc.status as status,
-- time
cc.datetime as start_at,
cc.datetime + (cc.duration::text || ' seconds')::interval as end_at,
(cc.duration::text || ' seconds')::interval as duration,
-- distance
cc.distance::float / 1000 as distance,
ST_Y(cc.start_position::geometry) as start_lat,
ST_X(cc.start_position::geometry) as start_lon,
ST_Y(cc.end_position::geometry) as end_lat,
ST_X(cc.end_position::geometry) as end_lon,
-- administration
cc.start_geo_code,
cc.end_geo_code,
-- operator data
cc.operator_class,
oo.name as operator,
cc.operator_journey_id,
id_p.operator_user_id as operator_passenger_id,
id_d.operator_user_id as operator_driver_id,
-- root carpool is from the passenger
-- root carpool is from the passenger
cc.cost as passenger_contribution,
cc_driver.cost as driver_revenue,
-- policy information
pi.policy_id as campaign_id,
-- an array of json objects with the incentives
-- to be split in numbered columns by the sql to csv exporter
-- (limit to 4 records)
jsonb_path_query_array(agg_incentives.incentive::jsonb, '$[0 to 3]') as incentive,
-- same for RPC calculated incentives (limit to 4 records)
jsonb_path_query_array(agg_incentives_rpc.incentive_rpc::jsonb, '$[0 to 3]') as incentive_rpc,
jsonb_path_query_array(agg_incentives_rpc.incentive_rpc::jsonb, '$[0 to 3]') as incentive_rpc
-- incentive_counterparts (limit to 2 records)
jsonb_path_query_array(agg_counterparts.incentive_counterpart::jsonb, '$[0 to 1]') as incentive_counterpart
-- jsonb_path_query_array(agg_counterparts.incentive_counterpart::jsonb, '$[0 to 1]') as incentive_counterpart
FROM carpool.carpools cc
-- join the driver's carpool
LEFT JOIN carpool.carpools cc_driver ON cc.acquisition_id = cc_driver.acquisition_id AND cc_driver.is_driver = TRUE
-- get operator data
LEFT JOIN operator.operators oo ON cc.operator_id = oo._id
-- get incentive from carpool.incentives
LEFT JOIN LATERAL (
SELECT json_agg(json_build_object(
Expand All @@ -68,57 +68,58 @@ export class CarpoolListQuery extends Query {
FROM carpool.incentives ci
WHERE ci.acquisition_id = cc.acquisition_id
) as agg_incentives ON TRUE
-- get RPC incentives from policy.incentives
LEFT JOIN LATERAL (
SELECT json_agg(json_build_object(
'amount', pi.amount
'campaign_id', pi_rpc.policy_id,
'amount', pi_rpc.amount
)) as incentive_rpc
FROM policy.incentives pi
WHERE pi.carpool_id = cc._id
FROM policy.incentives pi_rpc
WHERE pi_rpc.carpool_id = cc_driver._id
) as agg_incentives_rpc ON TRUE
-- get incentive_counterparts from carpool meta
LEFT JOIN LATERAL (
SELECT json_agg(y.counterpart) as incentive_counterpart
FROM (
SELECT json_build_object(
'target', x.target,
'amount', x.amount,
'siret', x.siret
) as counterpart
FROM json_to_recordset(cc.meta->'incentive_counterparts') x (target text, amount int, siret text)
WHERE x.target = 'passenger'
UNION ALL
SELECT json_build_object(
'target', x.target,
'amount', x.amount,
'siret', x.siret
) as counterpart
FROM json_to_recordset(cc_driver.meta->'incentive_counterparts') x (target text, amount int, siret text)
WHERE x.target = 'driver'
) y
) agg_counterparts ON true
-- LEFT JOIN LATERAL (
-- SELECT json_agg(y.counterpart) as incentive_counterpart
-- FROM (
-- SELECT json_build_object(
-- 'target', x.target,
-- 'amount', x.amount,
-- 'siret', x.siret
-- ) as counterpart
-- FROM json_to_recordset(cc.meta->'incentive_counterparts') x (target text, amount int, siret text)
-- WHERE x.target = 'passenger'
-- UNION ALL
-- SELECT json_build_object(
-- 'target', x.target,
-- 'amount', x.amount,
-- 'siret', x.siret
-- ) as counterpart
-- FROM json_to_recordset(cc_driver.meta->'incentive_counterparts') x (target text, amount int, siret text)
-- WHERE x.target = 'driver'
-- ) y
-- ) agg_counterparts ON true
-- identities
LEFT JOIN carpool.identities id_p ON cc.identity_id = id_p._id
LEFT JOIN carpool.identities id_d ON cc_driver.identity_id = id_d._id
-- join policies
LEFT JOIN policy.incentives pi ON cc_driver._id = pi.carpool_id
-- target the passenger for the root carpools
WHERE cc.is_driver = false
AND cc.datetime >= $1
AND cc.datetime < $2
-- TODO chunk carpools by datetime in the application code
ORDER BY cc.datetime DESC
LIMIT 10 -- TODO REMOVE THIS
),
-- select latest geo data for start and end geo codes only
-- move country code and name in their own columns
geo AS (
Expand All @@ -139,15 +140,15 @@ export class CarpoolListQuery extends Query {
WHERE arr IN (SELECT UNNEST(ARRAY[start_geo_code, end_geo_code]) FROM trips)
ORDER BY arr, year DESC
)
-- fields to export
SELECT
-- general trip identifiers
trips.trip_id,
trips.operator_journey_id,
trips.operator_class,
trips.status,
-- dates and times are in UTC
-- ceil times to 10 minutes and format for user's convenience
ts_ceil(trips.start_at, 600) as start_datetime_utc,
Expand All @@ -157,16 +158,16 @@ export class CarpoolListQuery extends Query {
to_char(ts_ceil(trips.end_at, 600), 'YYYY-MM-DD') as end_date_utc,
to_char(ts_ceil(trips.end_at, 600), 'HH24:MI:SS') as end_time_utc,
to_char(trips.duration, 'HH24:MI:SS') as duration,
-- distance in km with meter precision (float)
trips.distance,
-- truncate position depending on population density
trunc(trips.start_lat::numeric, gps.precision) as start_lat,
trunc(trips.start_lon::numeric, gps.precision) as start_lon,
trunc(trips.end_lat::numeric, gpe.precision) as end_lat,
trunc(trips.end_lon::numeric, gpe.precision) as end_lon,
-- administrative data
gps.arr as start_insee,
gps.l_arr as start_commune,
Expand All @@ -180,24 +181,40 @@ export class CarpoolListQuery extends Query {
gpe.l_epci as end_epci,
gpe.l_aom as end_aom,
gpe.l_country as end_pays,
-- operator data
trips.operator,
trips.operator_passenger_id,
trips.operator_driver_id,
-- financial data
trips.driver_revenue,
trips.passenger_contribution,
trips.campaign_id,
trips.incentive,
trips.incentive_rpc,
trips.incentive_counterpart,
-- incentives
trips.incentive[0]->>'index' as incentive_0_index,
trips.incentive[0]->>'siret' as incentive_0_siret,
trips.incentive[0]->>'amount' as incentive_0_amount,
trips.incentive[1]->>'index' as incentive_1_index,
trips.incentive[1]->>'siret' as incentive_1_siret,
trips.incentive[1]->>'amount' as incentive_1_amount,
trips.incentive[2]->>'index' as incentive_2_index,
trips.incentive[2]->>'siret' as incentive_2_siret,
trips.incentive[2]->>'amount' as incentive_2_amount,
-- RPC incentives
trips.incentive_rpc[0]->>'campaign_id' as incentive_rpc_0_campaign_id,
trips.incentive_rpc[0]->>'amount' as incentive_rpc_0_amount,
trips.incentive_rpc[1]->>'campaign_id' as incentive_rpc_1_campaign_id,
trips.incentive_rpc[1]->>'amount' as incentive_rpc_1_amount,
trips.incentive_rpc[2]->>'campaign_id' as incentive_rpc_2_campaign_id,
trips.incentive_rpc[2]->>'amount' as incentive_rpc_2_amount,
-- offer data (to be completed)
true as offer_public,
trips.start_at as offer_accepted_at -- TODO
FROM trips
LEFT JOIN geo AS gps ON trips.start_geo_code = gps.arr
LEFT JOIN geo AS gpe ON trips.end_geo_code = gpe.arr
Expand Down

0 comments on commit f638be5

Please sign in to comment.