From eec1f7ae39a958d0f752e5dcc271d4e136c6736c Mon Sep 17 00:00:00 2001 From: abernatskiy Date: Sun, 5 Nov 2023 04:14:53 +0900 Subject: [PATCH 1/4] Big-typed arrays support in codegen modified: typeorm/typeorm-codegen/src/codegen.ts --- typeorm/typeorm-codegen/src/codegen.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/typeorm/typeorm-codegen/src/codegen.ts b/typeorm/typeorm-codegen/src/codegen.ts index 6b53b44d0..af60196ec 100644 --- a/typeorm/typeorm-codegen/src/codegen.ts +++ b/typeorm/typeorm-codegen/src/codegen.ts @@ -140,13 +140,24 @@ export function generateOrmModels(model: Model, dir: OutDir): void { case 'list': switch(prop.type.item.type.kind) { case 'scalar': { + imports.useMarshal() let scalar = prop.type.item.type.name if (scalar == 'BigInt' || scalar == 'BigDecimal') { - throw new Error(`Property ${name}.${key} has unsupported type: can't generate code for native ${scalar} arrays.`) + out.line( + `@Column_("${getDbType(scalar)}", {transformer: {to: obj => ${marshalToJson( + prop, + 'obj' + )}, from: obj => ${marshalFromJson( + prop, + 'obj' + )}}, array: true, nullable: ${prop.nullable}})` + ) + } + else { + out.line( + `@Column_("${getDbType(scalar)}", {array: true, nullable: ${prop.nullable}})` + ) } - out.line( - `@Column_("${getDbType(scalar)}", {array: true, nullable: ${prop.nullable}})` - ) break } case 'enum': From cfc4931774ea7ff3d7d60f769a7d24fea149bda0 Mon Sep 17 00:00:00 2001 From: abernatskiy Date: Sun, 5 Nov 2023 17:46:25 +0900 Subject: [PATCH 2/4] Separate marshal.ts for typeorm-store tests The file was generated by @subsquid/typeorm-codegen@1.3.2 new file: typeorm/typeorm-store/src/test/lib/marshal.ts modified: typeorm/typeorm-store/src/test/lib/model.ts --- typeorm/typeorm-store/src/test/lib/marshal.ts | 179 ++++++++++++++++++ typeorm/typeorm-store/src/test/lib/model.ts | 4 +- 2 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 typeorm/typeorm-store/src/test/lib/marshal.ts diff --git a/typeorm/typeorm-store/src/test/lib/marshal.ts b/typeorm/typeorm-store/src/test/lib/marshal.ts new file mode 100644 index 000000000..ebfd9fa53 --- /dev/null +++ b/typeorm/typeorm-store/src/test/lib/marshal.ts @@ -0,0 +1,179 @@ +import assert from 'assert' + + +export interface Marshal { + fromJSON(value: unknown): T + toJSON(value: T): S +} + + +export const string: Marshal = { + fromJSON(value: unknown): string { + assert(typeof value === 'string', 'invalid String') + return value + }, + toJSON(value) { + return value + } +} + + +export const id = string + + +export const int: Marshal = { + fromJSON(value: unknown): number { + assert(Number.isInteger(value), 'invalid Int') + return value as number + }, + toJSON(value) { + return value + } +} + + +export const float: Marshal = { + fromJSON(value: unknown): number { + assert(typeof value === 'number', 'invalid Float') + return value as number + }, + toJSON(value) { + return value + } +} + + +export const boolean: Marshal = { + fromJSON(value: unknown): boolean { + assert(typeof value === 'boolean', 'invalid Boolean') + return value + }, + toJSON(value: boolean): boolean { + return value + } +} + + +export const bigint: Marshal = { + fromJSON(value: unknown): bigint { + assert(typeof value === 'string', `invalid BigInt ${value}`) + return BigInt(value) + }, + toJSON(value: bigint): string { + return value.toString() + } +} + + +export const bigdecimal: Marshal = { + fromJSON(value: unknown): bigint { + assert(typeof value === 'string', `invalid BigDecimal ${value}`) + return decimal.BigDecimal(value) + }, + toJSON(value: any): string { + return value.toString() + } +} + + +// credit - https://github.com/Urigo/graphql-scalars/blob/91b4ea8df891be8af7904cf84751930cc0c6613d/src/scalars/iso-date/validator.ts#L122 +const RFC_3339_REGEX = + /^(\d{4}-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60))(\.\d{1,})?([Z])$/ + + +function isIsoDateTimeString(s: string): boolean { + return RFC_3339_REGEX.test(s) +} + + +export const datetime: Marshal = { + fromJSON(value: unknown): Date { + assert(typeof value === 'string', 'invalid DateTime') + assert(isIsoDateTimeString(value), 'invalid DateTime') + return new Date(value) + }, + toJSON(value: Date): string { + return value.toISOString() + } +} + + +export const bytes: Marshal = { + fromJSON(value: unknown): Buffer { + assert(typeof value === 'string', 'invalid Bytes') + assert(value.length % 2 === 0, 'invalid Bytes') + assert(/^0x[0-9a-f]+$/i.test(value), 'invalid Bytes') + return Buffer.from(value.slice(2), 'hex') + }, + toJSON(value: Uint8Array): string { + if (Buffer.isBuffer(value)) { + return '0x' + value.toString('hex') + } else { + return '0x' + Buffer.from(value.buffer, value.byteOffset, value.byteLength).toString('hex') + } + } +} + + +export function fromList(list: unknown, f: (val: unknown) => T): T[] { + assert(Array.isArray(list)) + return list.map((val) => f(val)) +} + + +export function nonNull(val: T | undefined | null): T { + assert(val != null, 'non-nullable value is null') + return val +} + + +export const bigintTransformer = { + to(x?: bigint) { + return x?.toString() + }, + from(s?: string): bigint | undefined { + return s == null ? undefined : BigInt(s) + } +} + + +export const floatTransformer = { + to(x?: number) { + return x?.toString() + }, + from(s?: string): number | undefined { + return s == null ? undefined : Number(s) + } +} + + +export const bigdecimalTransformer = { + to(x?: any) { + return x?.toString() + }, + from(s?: any): any | undefined { + return s == null ? undefined : decimal.BigDecimal(s) + } +} + + +export function enumFromJson(json: unknown, enumObject: E): E[keyof E] { + assert(typeof json == 'string', 'invalid enum value') + let val = (enumObject as any)[json] + assert(typeof val == 'string', `invalid enum value`) + return val as any +} + + +const decimal = { + get BigDecimal(): any { + throw new Error('Package `@subsquid/big-decimal` is not installed') + } +} + + +try { + Object.defineProperty(decimal, "BigDecimal", { + value: require('@subsquid/big-decimal').BigDecimal + }) +} catch (e) {} diff --git a/typeorm/typeorm-store/src/test/lib/model.ts b/typeorm/typeorm-store/src/test/lib/model.ts index 8cd68bfc3..23214c908 100644 --- a/typeorm/typeorm-store/src/test/lib/model.ts +++ b/typeorm/typeorm-store/src/test/lib/model.ts @@ -1,5 +1,5 @@ import {Column as Column_, Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm' - +import * as marshal from './marshal' @Entity() export class Item { @@ -52,7 +52,7 @@ export class Data { @Column('int4', {array: true}) integerArray?: number[] | null - @Column('numeric', {transformer: {from: (s?: string) => s == null ? null : BigInt(s), to: (val?: bigint) => val?.toString()}}) + @Column('numeric', {transformer: marshal.bigintTransformer}) bigInteger?: bigint | null @Column('timestamp with time zone') From 6d199041adaeb30349b2936d46fcaa852ac4aa68 Mon Sep 17 00:00:00 2001 From: abernatskiy Date: Sun, 5 Nov 2023 18:22:54 +0900 Subject: [PATCH 3/4] Ensuring pg-types ^4.0.1 + adding the big-decimal dev dep to store modified: typeorm/typeorm-store/package.json --- typeorm/typeorm-store/package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/typeorm/typeorm-store/package.json b/typeorm/typeorm-store/package.json index 79861b8d0..e71306251 100644 --- a/typeorm/typeorm-store/package.json +++ b/typeorm/typeorm-store/package.json @@ -25,6 +25,7 @@ "typeorm": "^0.3.17" }, "devDependencies": { + "@subsquid/big-decimal": "^1.0.0", "@types/mocha": "^10.0.2", "@types/node": "^18.18.0", "@types/pg": "^8.10.3", @@ -32,5 +33,8 @@ "mocha": "^10.2.0", "pg": "^8.11.3", "typescript": "~5.2.2" + }, + "overrides": { + "pg-types": "^4.0.1" } } From 3627288666db7829a973384d99fb3644f2eed4e6 Mon Sep 17 00:00:00 2001 From: abernatskiy Date: Sun, 5 Nov 2023 18:25:07 +0900 Subject: [PATCH 4/4] Tests for scalar BigDecimal + Big* arrays modified: typeorm/typeorm-store/src/test/database.test.ts modified: typeorm/typeorm-store/src/test/lib/model.ts --- .../typeorm-store/src/test/database.test.ts | 50 ++++++++++++++++++- typeorm/typeorm-store/src/test/lib/model.ts | 10 ++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/typeorm/typeorm-store/src/test/database.test.ts b/typeorm/typeorm-store/src/test/database.test.ts index 48ffbf1c4..d9e22a37f 100644 --- a/typeorm/typeorm-store/src/test/database.test.ts +++ b/typeorm/typeorm-store/src/test/database.test.ts @@ -2,7 +2,7 @@ import expect from 'expect' import {TypeormDatabase} from '../database' import {Data} from './lib/model' import {getEntityManager, useDatabase} from './util' - +import {BigDecimal} from '@subsquid/big-decimal' describe('TypeormDatabase', function() { useDatabase([ @@ -14,6 +14,9 @@ describe('TypeormDatabase', function() { "integer" int4, integer_array int4[], big_integer numeric, + big_integer_array numeric[], + big_decimal numeric, + big_decimal_array numeric[], date_time timestamp with time zone, "bytes" bytea, "json" jsonb, @@ -107,6 +110,15 @@ describe('TypeormDatabase', function() { integer: 1, integerArray: [1, 10], bigInteger: 1000000000000000000000000000000000000000000000000000000000n, + bigIntegerArray: [ + 2000000000000000000000000000000000000000000000000000000000n, + 3000000000000000000000000000000000000000000000000000000000n + ], + bigDecimal: BigDecimal('43253426475865456435643564536356134564567533.72672356135613464526524234324243343261246235675'), + bigDecimalArray: [ + BigDecimal('236565735845787485246368356787532867985602345613578235689743524678458356783.34567356782458624671357365872456724562356256253'), + BigDecimal('324234566543745687485747875427365743676.46547364837584758584578') + ], dateTime: new Date(1000000000000), bytes: Buffer.from([100, 100, 100]), json: [1, {foo: 'bar'}] @@ -119,6 +131,15 @@ describe('TypeormDatabase', function() { integer: 2, integerArray: [2, 20], bigInteger: 2000000000000000000000000000000000000000000000000000000000n, + bigIntegerArray: [ + 1000000000000000000000000000000000000000000000000000000000n, + 3000000000000000000000000000000000000000000000000000000000n + ], + bigDecimal: BigDecimal('236565735845787485246368356787532867985602345613578235689743524678458356783.34567356782458624671357365872456724562356256253'), + bigDecimalArray: [ + BigDecimal('43253426475865456435643564536356134564567533.72672356135613464526524234324243343261246235675'), + BigDecimal('324234566543745687485747875427365743676.46547364837584758584578') + ], dateTime: new Date(2000000000000), bytes: Buffer.from([200, 200, 200]), json: [2, {foo: 'baz'}] @@ -131,6 +152,15 @@ describe('TypeormDatabase', function() { integer: 30, integerArray: [30, 300], bigInteger: 3000000000000000000000000000000000000000000000000000000000n, + bigIntegerArray: [ + 1000000000000000000000000000000000000000000000000000000000n, + 2000000000000000000000000000000000000000000000000000000000n + ], + bigDecimal: BigDecimal('324234566543745687485747875427365743676.46547364837584758584578'), + bigDecimalArray: [ + BigDecimal('43253426475865456435643564536356134564567533.72672356135613464526524234324243343261246235675'), + BigDecimal('236565735845787485246368356787532867985602345613578235689743524678458356783.34567356782458624671357365872456724562356256253') + ], dateTime: new Date(3000000000000), bytes: Buffer.from([3, 3, 3]), json: [3, {foo: 'qux'}] @@ -165,6 +195,15 @@ describe('TypeormDatabase', function() { integer: 10, integerArray: [10, 100], bigInteger: 8000000000000000000000000000000000000000000000000000000000_000_000n, + bigIntegerArray: [ + 9000000000000000000000000000000000000000000000000000000000_000n, + 8000000000000000000000000000000000000000000000000000000000_000_000n + ], + bigDecimal: BigDecimal('245246738564679435764568753895478567947568346745672456247624565475648456.245724686358463734567348657524376437'), + bigDecimalArray: [ + BigDecimal('6457567436813456435724623456437456843512346452874845614363275427254.3465473262456'), + BigDecimal('245246738564679435764568753895478567947568346745672456247624565475648456.245724686358463734567348657524376437') + ], dateTime: new Date(100000), bytes: Buffer.from([1, 1, 1]), json: ["b1", {foo: 'bar'}] @@ -177,6 +216,15 @@ describe('TypeormDatabase', function() { integer: 20, integerArray: [20, 200], bigInteger: 9000000000000000000000000000000000000000000000000000000000_000n, + bigIntegerArray: [ + 8000000000000000000000000000000000000000000000000000000000_000_000n, + 9000000000000000000000000000000000000000000000000000000000_000n + ], + bigDecimal: BigDecimal('6457567436813456435724623456437456843512346452874845614363275427254.3465473262456'), + bigDecimalArray: [ + BigDecimal('245246738564679435764568753895478567947568346745672456247624565475648456.245724686358463734567348657524376437'), + BigDecimal('6457567436813456435724623456437456843512346452874845614363275427254.3465473262456') + ], dateTime: new Date(2000), bytes: Buffer.from([2, 2, 2]), json: {b2: true} diff --git a/typeorm/typeorm-store/src/test/lib/model.ts b/typeorm/typeorm-store/src/test/lib/model.ts index 23214c908..0c51782f0 100644 --- a/typeorm/typeorm-store/src/test/lib/model.ts +++ b/typeorm/typeorm-store/src/test/lib/model.ts @@ -1,4 +1,5 @@ import {Column as Column_, Column, Entity, ManyToOne, PrimaryColumn} from 'typeorm' +import {BigDecimal} from '@subsquid/big-decimal' import * as marshal from './marshal' @Entity() @@ -55,6 +56,15 @@ export class Data { @Column('numeric', {transformer: marshal.bigintTransformer}) bigInteger?: bigint | null + @Column_("numeric", {transformer: {to: obj => obj == null ? undefined : obj.map((val: any) => val == null ? undefined : marshal.bigint.toJSON(val)), from: obj => obj == null ? undefined : marshal.fromList(obj, val => val == null ? undefined : marshal.bigint.fromJSON(val))}, array: true, nullable: true}) + bigIntegerArray?: bigint[] | null + + @Column('numeric', {transformer: marshal.bigdecimalTransformer}) + bigDecimal?: BigDecimal | null + + @Column_("numeric", {transformer: {to: obj => obj == null ? undefined : obj.map((val: any) => val == null ? undefined : marshal.bigdecimal.toJSON(val)), from: obj => obj == null ? undefined : marshal.fromList(obj, val => val == null ? undefined : marshal.bigdecimal.fromJSON(val))}, array: true, nullable: true}) + bigDecimalArray?: BigDecimal[] | null + @Column('timestamp with time zone') dateTime?: Date | null