From 8e1995dd95ce46288b0eea4b187105dffdb8fe77 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 6 Jan 2025 16:31:54 +0000 Subject: [PATCH 1/8] Make improvments to hybrid ux: - add `aggregate.hybrid` method - support `maxVectorDistance` in this new method - fix minor typing issues when doing grouped aggregates - add integration tests for aggregate search queries --- src/collections/aggregate/index.ts | 109 +++++++++++++++--- src/collections/aggregate/integration.test.ts | 84 ++++++++++++++ src/graphql/aggregator.ts | 15 +++ src/graphql/hybrid.ts | 7 ++ 4 files changed, 198 insertions(+), 17 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index 63a41496..f67e89f6 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -7,7 +7,8 @@ import { FilterValue } from '../filters/index.js'; import { WeaviateQueryError } from '../../errors.js'; import { Aggregator } from '../../graphql/index.js'; -import { toBase64FromMedia } from '../../index.js'; +import { PrimitiveKeys, toBase64FromMedia } from '../../index.js'; +import { Bm25QueryProperty } from '../query/types.js'; import { Serialize } from '../serialize/index.js'; export type AggregateBaseOptions = { @@ -35,6 +36,19 @@ export type AggregateNearOptions = AggregateBaseOptions & { targetVector?: string; }; +export type AggregateHybridOptions = AggregateBaseOptions & { + alpha?: number; + maxVectorDistance?: number; + objectLimit?: number; + queryProperties?: (PrimitiveKeys | Bm25QueryProperty)[]; + targetVector?: string; + vector?: number[]; +}; + +export type AggregateGroupByHybridOptions = AggregateHybridOptions & { + groupBy: (keyof T & string) | GroupByAggregate; +}; + export type AggregateGroupByNearOptions = AggregateNearOptions & { groupBy: (keyof T & string) | GroupByAggregate; }; @@ -346,9 +360,26 @@ class AggregateManager implements Aggregate { this.tenant = tenant; this.groupBy = { + hybrid: | undefined = undefined>( + query: string, + opts: AggregateGroupByHybridOptions + ): Promise[]> => { + let builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withHybrid({ + query: query, + alpha: opts?.alpha, + maxVectorDistance: opts?.maxVectorDistance, + properties: opts?.queryProperties as string[], + targetVectors: opts?.targetVector ? [opts.targetVector] : undefined, + vector: opts?.vector, + }); + if (opts?.objectLimit) { + builder = builder.withObjectLimit(opts.objectLimit); + } + return this.doGroupBy(builder); + }, nearImage: async | undefined = undefined>( image: string | Buffer, - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]> => { const builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withNearImage({ image: await toBase64FromMedia(image), @@ -363,7 +394,7 @@ class AggregateManager implements Aggregate { }, nearObject: | undefined = undefined>( id: string, - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]> => { const builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withNearObject({ id: id, @@ -378,7 +409,7 @@ class AggregateManager implements Aggregate { }, nearText: | undefined = undefined>( query: string | string[], - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]> => { const builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withNearText({ concepts: Array.isArray(query) ? query : [query], @@ -393,7 +424,7 @@ class AggregateManager implements Aggregate { }, nearVector: | undefined = undefined>( vector: number[], - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]> => { const builder = this.base(opts?.returnMetrics, opts?.filters, opts?.groupBy).withNearVector({ vector: vector, @@ -489,6 +520,24 @@ class AggregateManager implements Aggregate { return new AggregateManager(connection, name, dbVersionSupport, consistencyLevel, tenant); } + hybrid>( + query: string, + opts?: AggregateHybridOptions + ): Promise> { + let builder = this.base(opts?.returnMetrics, opts?.filters).withHybrid({ + query: query, + alpha: opts?.alpha, + maxVectorDistance: opts?.maxVectorDistance, + properties: opts?.queryProperties as string[], + targetVectors: opts?.targetVector ? [opts.targetVector] : undefined, + vector: opts?.vector, + }); + if (opts?.objectLimit) { + builder = builder.withObjectLimit(opts.objectLimit); + } + return this.do(builder); + } + async nearImage>( image: string | Buffer, opts?: AggregateNearOptions @@ -602,6 +651,19 @@ class AggregateManager implements Aggregate { export interface Aggregate { /** This namespace contains methods perform a group by search while aggregating metrics. */ groupBy: AggregateGroupBy; + /** + * Aggregate metrics over the objects returned by a hybrid search on this collection. + * + * This method requires that the objects in the collection have associated vectors. + * + * @param {string} query The text query to search for. + * @param {AggregateHybridOptions} opts The options for the request. + * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. + */ + hybrid>( + query: string, + opts?: AggregateHybridOptions + ): Promise>; /** * Aggregate metrics over the objects returned by a near image vector search on this collection. * @@ -673,44 +735,57 @@ export interface Aggregate { export interface AggregateGroupBy { /** - * Aggregate metrics over the objects returned by a near image vector search on this collection. + * Aggregate metrics over the objects grouped by a specified property and returned by a hybrid search on this collection. + * + * This method requires that the objects in the collection have associated vectors. + * + * @param {string} query The text query to search for. + * @param {AggregateGroupByHybridOptions} opts The options for the request. + * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. + */ + hybrid>( + query: string, + opts: AggregateGroupByHybridOptions + ): Promise[]>; + /** + * Aggregate metrics over the objects grouped by a specified property and returned by a near image vector search on this collection. * * At least one of `certainty`, `distance`, or `object_limit` must be specified here for the vector search. * * This method requires a vectorizer capable of handling base64-encoded images, e.g. `img2vec-neural`, `multi2vec-clip`, and `multi2vec-bind`. * * @param {string | Buffer} image The image to search on. This can be a base64 string, a file path string, or a buffer. - * @param {AggregateGroupByNearOptions} [opts] The options for the request. + * @param {AggregateGroupByNearOptions} opts The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. */ nearImage>( image: string | Buffer, - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]>; /** - * Aggregate metrics over the objects returned by a near object search on this collection. + * Aggregate metrics over the objects grouped by a specified property and returned by a near object search on this collection. * * At least one of `certainty`, `distance`, or `object_limit` must be specified here for the vector search. * * This method requires that the objects in the collection have associated vectors. * * @param {string} id The ID of the object to search for. - * @param {AggregateGroupByNearOptions} [opts] The options for the request. + * @param {AggregateGroupByNearOptions} opts The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. */ nearObject>( id: string, - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]>; /** - * Aggregate metrics over the objects returned by a near text vector search on this collection. + * Aggregate metrics over the objects grouped by a specified property and returned by a near text vector search on this collection. * * At least one of `certainty`, `distance`, or `object_limit` must be specified here for the vector search. * * This method requires a vectorizer capable of handling text, e.g. `text2vec-contextionary`, `text2vec-openai`, etc. * * @param {string | string[]} query The text to search for. - * @param {AggregateGroupByNearOptions} [opts] The options for the request. + * @param {AggregateGroupByNearOptions} opts The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. */ nearText>( @@ -718,22 +793,22 @@ export interface AggregateGroupBy { opts: AggregateGroupByNearOptions ): Promise[]>; /** - * Aggregate metrics over the objects returned by a near vector search on this collection. + * Aggregate metrics over the objects grouped by a specified property and returned by a near vector search on this collection. * * At least one of `certainty`, `distance`, or `object_limit` must be specified here for the vector search. * * This method requires that the objects in the collection have associated vectors. * * @param {number[]} vector The vector to search for. - * @param {AggregateGroupByNearOptions} [opts] The options for the request. + * @param {AggregateGroupByNearOptions} opts The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects returned by the vector search. */ nearVector>( vector: number[], - opts?: AggregateGroupByNearOptions + opts: AggregateGroupByNearOptions ): Promise[]>; /** - * Aggregate metrics over all the objects in this collection without any vector search. + * Aggregate metrics over all the objects in this collection grouped by a specified property without any vector search. * * @param {AggregateGroupByOptions} [opts] The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects in the collection. diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index b7b3a418..577b765a 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -355,6 +355,7 @@ describe('Testing of collection.aggregate.overAll with a multi-tenancy collectio beforeAll(async () => { client = await weaviate.connectToLocal(); + collection = client.collections.get(collectionName); return client.collections .create({ name: collectionName, @@ -389,3 +390,86 @@ describe('Testing of collection.aggregate.overAll with a multi-tenancy collectio WeaviateQueryError )); }); + +describe('Testing of collection.aggregate search methods', () => { + let client: WeaviateClient; + let collection: Collection; + const collectionName = 'TestCollectionAggregateSearches'; + + let uuid: string; + + afterAll(async () => { + return (await client).collections.delete(collectionName).catch((err) => { + console.error(err); + throw err; + }); + }); + + beforeAll(async () => { + client = await weaviate.connectToLocal(); + collection = client.collections.get(collectionName); + return client.collections + .create({ + name: collectionName, + properties: [ + { + name: 'text', + dataType: 'text', + }, + ], + vectorizers: weaviate.configure.vectorizer.text2VecContextionary(), + }) + .then(async () => { + const data: Array = []; + for (let i = 0; i < 100; i++) { + data.push({ + properties: { + text: 'test', + }, + }); + } + await collection.data.insertMany(data).then((res) => { + uuid = res.uuids[0]; + }); + }); + }); + + it('should return an aggregation on a hybrid search', async () => { + const result = await collection.aggregate.hybrid('test', { + alpha: 0.5, + maxVectorDistance: 0, + queryProperties: ['text'], + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.totalCount).toEqual(100); + expect(result.properties.text.count).toEqual(100); + }); + + it('should return an aggregation on a nearText search', async () => { + const result = await collection.aggregate.nearText('test', { + objectLimit: 100, + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.totalCount).toEqual(100); + expect(result.properties.text.count).toEqual(100); + }); + + it('should return an aggregation on a nearVector search', async () => { + const obj = await collection.query.fetchObjectById(uuid, { includeVector: true }); + const result = await collection.aggregate.nearVector(obj?.vectors.default!, { + objectLimit: 100, + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.totalCount).toEqual(100); + expect(result.properties.text.count).toEqual(100); + }); + + it('should return an aggregation on a nearObject search', async () => { + const result = await collection.aggregate.nearObject(uuid, { + objectLimit: 100, + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.totalCount).toEqual(100); + expect(result.properties.text.count).toEqual(100); + }); +}); diff --git a/src/graphql/aggregator.ts b/src/graphql/aggregator.ts index aaab6697..58ffb615 100644 --- a/src/graphql/aggregator.ts +++ b/src/graphql/aggregator.ts @@ -2,6 +2,7 @@ import Connection from '../connection/index.js'; import { WhereFilter } from '../openapi/types.js'; import { CommandBase } from '../validation/commandBase.js'; import { isValidPositiveIntProperty } from '../validation/number.js'; +import Hybrid, { HybridArgs } from './hybrid.js'; import NearMedia, { NearAudioArgs, NearDepthArgs, @@ -24,6 +25,7 @@ export default class Aggregator extends CommandBase { private className?: string; private fields?: string; private groupBy?: string[]; + private hybridString?: string; private includesNearMediaFilter: boolean; private limit?: number; private nearMediaString?: string; @@ -133,6 +135,15 @@ export default class Aggregator extends CommandBase { return this; }; + withHybrid = (args: HybridArgs) => { + try { + this.hybridString = new Hybrid(args).toString(); + } catch (e: any) { + this.addError(e.toString()); + } + return this; + }; + withObjectLimit = (objectLimit: number) => { if (!isValidPositiveIntProperty(objectLimit)) { throw new Error('objectLimit must be a non-negative integer'); @@ -222,6 +233,10 @@ export default class Aggregator extends CommandBase { args = [...args, `groupBy:${JSON.stringify(this.groupBy)}`]; } + if (this.hybridString) { + args = [...args, `hybrid:${this.hybridString}`]; + } + if (this.limit) { args = [...args, `limit:${this.limit}`]; } diff --git a/src/graphql/hybrid.ts b/src/graphql/hybrid.ts index b344041e..04a9de53 100644 --- a/src/graphql/hybrid.ts +++ b/src/graphql/hybrid.ts @@ -8,6 +8,7 @@ export interface HybridArgs { targetVectors?: string[]; fusionType?: FusionType; searches?: HybridSubSearch[]; + maxVectorDistance?: number; } export interface NearTextSubSearch { @@ -87,6 +88,7 @@ export default class GraphQLHybrid { private targetVectors?: string[]; private fusionType?: FusionType; private searches?: GraphQLHybridSubSearch[]; + private maxVectorDistance?: number; constructor(args: HybridArgs) { this.alpha = args.alpha; @@ -96,6 +98,7 @@ export default class GraphQLHybrid { this.targetVectors = args.targetVectors; this.fusionType = args.fusionType; this.searches = args.searches?.map((search) => new GraphQLHybridSubSearch(search)); + this.maxVectorDistance = args.maxVectorDistance; } toString() { @@ -125,6 +128,10 @@ export default class GraphQLHybrid { args = [...args, `searches:[${this.searches.map((search) => search.toString()).join(',')}]`]; } + if (this.maxVectorDistance !== undefined) { + args = [...args, `maxVectorDistance:${this.maxVectorDistance}`]; + } + return `{${args.join(',')}}`; } } From 67a9cf225d206acd9c034ceb5bf51cadb19ef27b Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 6 Jan 2025 16:32:21 +0000 Subject: [PATCH 2/8] Add missed changes --- src/collections/aggregate/index.ts | 26 ++++--- src/collections/aggregate/integration.test.ts | 76 ++++++++++++++----- 2 files changed, 73 insertions(+), 29 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index f67e89f6..c88df849 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -16,18 +16,16 @@ export type AggregateBaseOptions = { returnMetrics?: M; }; -export type AggregateGroupByOptions = AggregateOptions & { - groupBy: (keyof T & string) | GroupByAggregate; +export type AggregateGroupByOptions = AggregateBaseOptions & { + groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; }; export type GroupByAggregate = { - property: keyof T & string; + property: T extends undefined ? string : keyof T & string; limit?: number; }; -export type AggregateOptions = AggregateBaseOptions; - -export type AggregateBaseOverAllOptions = AggregateBaseOptions; +export type AggregateOverAllOptions = AggregateBaseOptions; export type AggregateNearOptions = AggregateBaseOptions & { certainty?: number; @@ -46,11 +44,11 @@ export type AggregateHybridOptions = AggregateBaseOptions & { }; export type AggregateGroupByHybridOptions = AggregateHybridOptions & { - groupBy: (keyof T & string) | GroupByAggregate; + groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; }; export type AggregateGroupByNearOptions = AggregateNearOptions & { - groupBy: (keyof T & string) | GroupByAggregate; + groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; }; export type AggregateBoolean = { @@ -453,7 +451,7 @@ class AggregateManager implements Aggregate { base( metrics?: PropertiesMetrics, filters?: FilterValue, - groupBy?: (keyof T & string) | GroupByAggregate + groupBy?: (keyof T & string) | GroupByAggregate | string ) { let fields = 'meta { count }'; let builder = this.query().withClassName(this.name); @@ -602,7 +600,9 @@ class AggregateManager implements Aggregate { return this.do(builder); } - overAll>(opts?: AggregateOptions): Promise> { + overAll>( + opts?: AggregateOverAllOptions + ): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters); return this.do(builder); } @@ -637,7 +637,7 @@ class AggregateManager implements Aggregate { prop: groupedBy.path[0], value: groupedBy.value, }, - properties: rest.length > 0 ? rest : undefined, + properties: rest, totalCount: meta?.count, }; }) @@ -730,7 +730,9 @@ export interface Aggregate { * @param {AggregateOptions} [opts] The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects in the collection. */ - overAll>(opts?: AggregateOptions): Promise>; + overAll>( + opts?: AggregateOverAllOptions + ): Promise>; } export interface AggregateGroupBy { diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index 577b765a..b493359c 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -140,7 +140,7 @@ describe('Testing of the collection.aggregate methods', () => { expect(result[0].totalCount).toEqual(100); expect(result[0].groupedBy.prop).toEqual('text'); expect(result[0].groupedBy.value).toEqual('test'); - expect(result[0].properties).toBeUndefined(); + expect(result[0].properties).toEqual({}); }); it('should aggregate grouped by data with a near text search and no property metrics', async () => { @@ -152,7 +152,7 @@ describe('Testing of the collection.aggregate methods', () => { expect(result[0].totalCount).toEqual(100); expect(result[0].groupedBy.prop).toEqual('text'); expect(result[0].groupedBy.value).toEqual('test'); - expect(result[0].properties).toBeUndefined(); + expect(result[0].properties).toEqual({}); }); it('should aggregate data without a search and one generic property metric', async () => { @@ -419,7 +419,7 @@ describe('Testing of collection.aggregate search methods', () => { ], vectorizers: weaviate.configure.vectorizer.text2VecContextionary(), }) - .then(async () => { + .then(() => { const data: Array = []; for (let i = 0; i < 100; i++) { data.push({ @@ -428,9 +428,10 @@ describe('Testing of collection.aggregate search methods', () => { }, }); } - await collection.data.insertMany(data).then((res) => { - uuid = res.uuids[0]; - }); + return collection.data.insertMany(data); + }) + .then((res) => { + uuid = res.uuids[0]; }); }); @@ -441,35 +442,76 @@ describe('Testing of collection.aggregate search methods', () => { queryProperties: ['text'], returnMetrics: collection.metrics.aggregate('text').text(['count']), }); - expect(result.totalCount).toEqual(100); - expect(result.properties.text.count).toEqual(100); + expect(result.totalCount).toBeGreaterThan(0); + }); + + it('should return a grouped aggregation on a hybrid search', async () => { + const result = await collection.aggregate.groupBy.hybrid('test', { + objectLimit: 1000, + groupBy: 'text', + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.length).toEqual(1); + expect(result[0].groupedBy.prop).toEqual('text'); + expect(result[0].groupedBy.value).toEqual('test'); }); it('should return an aggregation on a nearText search', async () => { const result = await collection.aggregate.nearText('test', { - objectLimit: 100, + objectLimit: 1000, returnMetrics: collection.metrics.aggregate('text').text(['count']), }); - expect(result.totalCount).toEqual(100); - expect(result.properties.text.count).toEqual(100); + expect(result.totalCount).toBeGreaterThan(0); + }); + + it('should return a grouped aggregation on a nearText search', async () => { + const result = await collection.aggregate.groupBy.nearText('test', { + objectLimit: 1000, + groupBy: 'text', + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.length).toEqual(1); + expect(result[0].groupedBy.prop).toEqual('text'); + expect(result[0].groupedBy.value).toEqual('test'); }); it('should return an aggregation on a nearVector search', async () => { const obj = await collection.query.fetchObjectById(uuid, { includeVector: true }); const result = await collection.aggregate.nearVector(obj?.vectors.default!, { - objectLimit: 100, + objectLimit: 1000, returnMetrics: collection.metrics.aggregate('text').text(['count']), }); - expect(result.totalCount).toEqual(100); - expect(result.properties.text.count).toEqual(100); + expect(result.totalCount).toBeGreaterThan(0); + }); + + it('should return a grouped aggregation on a nearVector search', async () => { + const obj = await collection.query.fetchObjectById(uuid, { includeVector: true }); + const result = await collection.aggregate.groupBy.nearVector(obj?.vectors.default!, { + objectLimit: 1000, + groupBy: 'text', + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.length).toEqual(1); + expect(result[0].groupedBy.prop).toEqual('text'); + expect(result[0].groupedBy.value).toEqual('test'); }); it('should return an aggregation on a nearObject search', async () => { const result = await collection.aggregate.nearObject(uuid, { - objectLimit: 100, + objectLimit: 1000, returnMetrics: collection.metrics.aggregate('text').text(['count']), }); - expect(result.totalCount).toEqual(100); - expect(result.properties.text.count).toEqual(100); + expect(result.totalCount).toBeGreaterThan(0); + }); + + it('should return a grouped aggregation on a nearText search', async () => { + const result = await collection.aggregate.groupBy.nearObject(uuid, { + objectLimit: 1000, + groupBy: 'text', + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.length).toEqual(1); + expect(result[0].groupedBy.prop).toEqual('text'); + expect(result[0].groupedBy.value).toEqual('test'); }); }); From 45d88acc3244086d3bf7e7761b7597cb2b156e3a Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 8 Jan 2025 12:11:56 +0000 Subject: [PATCH 3/8] Skip aggregate hybrid tests for 1.24.26 --- src/collections/aggregate/integration.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index b493359c..de636f92 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -436,6 +436,10 @@ describe('Testing of collection.aggregate search methods', () => { }); it('should return an aggregation on a hybrid search', async () => { + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0))) { + console.warn('Skipping test as there is a bug with this in 1.24.26 that will not be fixed'); + return; + } const result = await collection.aggregate.hybrid('test', { alpha: 0.5, maxVectorDistance: 0, @@ -446,6 +450,10 @@ describe('Testing of collection.aggregate search methods', () => { }); it('should return a grouped aggregation on a hybrid search', async () => { + if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0))) { + console.warn('Skipping test as there is a bug with this in 1.24.26 that will not be fixed'); + return; + } const result = await collection.aggregate.groupBy.hybrid('test', { objectLimit: 1000, groupBy: 'text', From 878e1d1ca229796a02ba6b8eb122893ec85765b9 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 8 Jan 2025 12:12:48 +0000 Subject: [PATCH 4/8] Bump image tags in CI --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index afa58880..5344b274 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,10 +8,10 @@ on: env: WEAVIATE_124: 1.24.26 - WEAVIATE_125: 1.25.28 + WEAVIATE_125: 1.25.29 WEAVIATE_126: 1.26.13 - WEAVIATE_127: 1.27.8 - WEAVIATE_128: 1.28.1-77a2178 + WEAVIATE_127: 1.27.9 + WEAVIATE_128: 1.28.2 jobs: checks: From 83dfde6f196331b2af82028b043bddaf6d8271d4 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Wed, 8 Jan 2025 12:27:55 +0000 Subject: [PATCH 5/8] Fix bad tag in CI --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5344b274..4124c0c5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,7 +8,7 @@ on: env: WEAVIATE_124: 1.24.26 - WEAVIATE_125: 1.25.29 + WEAVIATE_125: 1.25.28 WEAVIATE_126: 1.26.13 WEAVIATE_127: 1.27.9 WEAVIATE_128: 1.28.2 From 0a2ade06348f59244a5bd76059a0b43eb80f8c56 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 13 Jan 2025 14:46:06 +0100 Subject: [PATCH 6/8] Make typing changes in response to review --- src/collections/aggregate/index.ts | 44 ++++++++++++++---------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index c88df849..db31246a 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -11,13 +11,15 @@ import { PrimitiveKeys, toBase64FromMedia } from '../../index.js'; import { Bm25QueryProperty } from '../query/types.js'; import { Serialize } from '../serialize/index.js'; -export type AggregateBaseOptions = { +export type AggregateBaseOptions = { filters?: FilterValue; returnMetrics?: M; }; -export type AggregateGroupByOptions = AggregateBaseOptions & { - groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; +export type PropertyOf = T extends undefined ? string : keyof T & string; + +export type AggregateGroupByOptions = AggregateBaseOptions & { + groupBy: PropertyOf | GroupByAggregate; }; export type GroupByAggregate = { @@ -25,16 +27,16 @@ export type GroupByAggregate = { limit?: number; }; -export type AggregateOverAllOptions = AggregateBaseOptions; +export type AggregateOverAllOptions = AggregateBaseOptions; -export type AggregateNearOptions = AggregateBaseOptions & { +export type AggregateNearOptions = AggregateBaseOptions & { certainty?: number; distance?: number; objectLimit?: number; targetVector?: string; }; -export type AggregateHybridOptions = AggregateBaseOptions & { +export type AggregateHybridOptions = AggregateBaseOptions & { alpha?: number; maxVectorDistance?: number; objectLimit?: number; @@ -44,11 +46,11 @@ export type AggregateHybridOptions = AggregateBaseOptions & { }; export type AggregateGroupByHybridOptions = AggregateHybridOptions & { - groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; + groupBy: PropertyOf | GroupByAggregate; }; -export type AggregateGroupByNearOptions = AggregateNearOptions & { - groupBy: T extends undefined ? string : (keyof T & string) | GroupByAggregate; +export type AggregateGroupByNearOptions = AggregateNearOptions & { + groupBy: PropertyOf | GroupByAggregate; }; export type AggregateBoolean = { @@ -538,7 +540,7 @@ class AggregateManager implements Aggregate { async nearImage>( image: string | Buffer, - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters).withNearImage({ image: await toBase64FromMedia(image), @@ -554,7 +556,7 @@ class AggregateManager implements Aggregate { nearObject>( id: string, - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters).withNearObject({ id: id, @@ -570,7 +572,7 @@ class AggregateManager implements Aggregate { nearText>( query: string | string[], - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters).withNearText({ concepts: Array.isArray(query) ? query : [query], @@ -586,7 +588,7 @@ class AggregateManager implements Aggregate { nearVector>( vector: number[], - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters).withNearVector({ vector: vector, @@ -600,9 +602,7 @@ class AggregateManager implements Aggregate { return this.do(builder); } - overAll>( - opts?: AggregateOverAllOptions - ): Promise> { + overAll>(opts?: AggregateOverAllOptions): Promise> { const builder = this.base(opts?.returnMetrics, opts?.filters); return this.do(builder); } @@ -677,7 +677,7 @@ export interface Aggregate { */ nearImage>( image: string | Buffer, - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise>; /** * Aggregate metrics over the objects returned by a near object search on this collection. @@ -692,7 +692,7 @@ export interface Aggregate { */ nearObject>( id: string, - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise>; /** * Aggregate metrics over the objects returned by a near vector search on this collection. @@ -707,7 +707,7 @@ export interface Aggregate { */ nearText>( query: string | string[], - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise>; /** * Aggregate metrics over the objects returned by a near vector search on this collection. @@ -722,7 +722,7 @@ export interface Aggregate { */ nearVector>( vector: number[], - opts?: AggregateNearOptions + opts?: AggregateNearOptions ): Promise>; /** * Aggregate metrics over all the objects in this collection without any vector search. @@ -730,9 +730,7 @@ export interface Aggregate { * @param {AggregateOptions} [opts] The options for the request. * @returns {Promise[]>} The aggregated metrics for the objects in the collection. */ - overAll>( - opts?: AggregateOverAllOptions - ): Promise>; + overAll>(opts?: AggregateOverAllOptions): Promise>; } export interface AggregateGroupBy { From c2bba8f3c6da5dc8579bda0c85896ea5e03899f6 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 13 Jan 2025 14:56:04 +0100 Subject: [PATCH 7/8] Use `PropertyOf` in `aggregate.base()` signature --- src/collections/aggregate/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index db31246a..c91358a3 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -450,11 +450,7 @@ class AggregateManager implements Aggregate { return new Aggregator(this.connection); } - base( - metrics?: PropertiesMetrics, - filters?: FilterValue, - groupBy?: (keyof T & string) | GroupByAggregate | string - ) { + base(metrics?: PropertiesMetrics, filters?: FilterValue, groupBy?: PropertyOf | GroupByAggregate) { let fields = 'meta { count }'; let builder = this.query().withClassName(this.name); if (metrics) { From fea9b0d60c48ef128d17032719d987cbe3003b7f Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 13 Jan 2025 15:05:11 +0100 Subject: [PATCH 8/8] Add missed changes from wrong branch push --- src/collections/aggregate/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index c91358a3..f038b93c 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -23,7 +23,7 @@ export type AggregateGroupByOptions = AggregateBaseOptions & { }; export type GroupByAggregate = { - property: T extends undefined ? string : keyof T & string; + property: PropertyOf; limit?: number; }; @@ -140,11 +140,11 @@ export type AggregateMetrics = { [K in keyof M]: M[K] extends true ? number : never; }; -export type MetricsProperty = T extends undefined ? string : keyof T & string; +export type MetricsProperty = PropertyOf; export const metrics = () => { return { - aggregate:

>(property: P) => new MetricsManager(property), + aggregate:

>(property: P) => new MetricsManager(property), }; }; @@ -157,10 +157,10 @@ export interface Metrics { See [the docs](https://weaviate.io/developers/weaviate/search/aggregate) for more details! */ - aggregate:

>(property: P) => MetricsManager; + aggregate:

>(property: P) => MetricsManager; } -export class MetricsManager> { +export class MetricsManager> { private propertyName: P; constructor(property: P) {