Skip to content

Commit

Permalink
Modified code such that in-memory-paging supports empty filter and ar…
Browse files Browse the repository at this point in the history 
…ray (#632)

* Modified code such that in-memory-paging supports empty filter and array

* Removed the need for FilterSelector
  • Loading branch information
@thehenrytsai
thehenrytsai authored Nov 29, 2023
1 parent d731b8b commit 3b6e68e
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 626 deletions.
78 changes: 60 additions & 18 deletions src/store/index-level.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,11 +304,15 @@ export class IndexLevel {
// we create a matches map so that we can short-circuit matched items within the async single query below.
const matches:Map<string, IndexedItem> = new Map();

// select the searchFilters
const searchFilters = FilterSelector.reduceFilters(filters);
// If the filter is empty, we just give it an empty filter so that we can iterate over all the items later in executeSingleFilterQuery().
// We could do the iteration here, but it would be duplicating the same logic, so decided to just setup the data structure here.
if (filters.length === 0) {
filters = [{}];
}

try {
await Promise.all(searchFilters.map(searchFilter => {
return this.executeSingleFilterQuery(tenant, searchFilter, filters, sortProperty, matches, options );
await Promise.all(filters.map(filter => {
return this.executeSingleFilterQuery(tenant, filter, sortProperty, matches, options );
}));
} catch (error) {
if ((error as DwnError).code === DwnErrorCode.IndexInvalidSortProperty) {
Expand Down Expand Up @@ -338,15 +342,23 @@ export class IndexLevel {
*/
private async executeSingleFilterQuery(
tenant: string,
searchFilter: Filter,
matchFilters: Filter[],
filter: Filter,
sortProperty: string,
matches: Map<string, IndexedItem>,
levelOptions?: IndexLevelOptions
): Promise<void> {

// Note: We have an array of Promises in order to support OR (anyOf) matches when given a list of accepted values for a property
const filterPromises: Promise<IndexedItem[]>[] = [];

// If the filter is empty, then we just iterate over one of the indexes that contains all the records and return all items.
if (isEmptyObject(filter)) {
const getAllItemsPromise = this.getAllItems(tenant, sortProperty);
filterPromises.push(getAllItemsPromise);
}

// else the filter is not empty
const searchFilter = FilterSelector.reduceFilter(filter);
for (const propertyName in searchFilter) {
const propertyFilter = searchFilter[propertyName];
// We will find the union of these many individual queries later.
Expand All @@ -370,12 +382,13 @@ export class IndexLevel {

// acting as an OR match for the property, any of the promises returning a match will be treated as a property match
for (const promise of filterPromises) {
const indexItems = await promise;
// reminder: the promise returns a list of IndexedItem satisfying a particular property match
for (const indexedItem of await promise) {
for (const indexedItem of indexItems) {
// short circuit: if a data is already included to the final matched key set (by a different `Filter`),
// no need to evaluate if the data satisfies this current filter being evaluated
// otherwise check that the item is a match.
if (matches.has(indexedItem.itemId) || !FilterUtility.matchAnyFilter(indexedItem.indexes, matchFilters)) {
if (matches.has(indexedItem.itemId) || !FilterUtility.matchFilter(indexedItem.indexes, filter)) {
continue;
}

Expand All @@ -389,6 +402,15 @@ export class IndexLevel {
}
}

private async getAllItems(tenant: string, sortProperty: string): Promise<IndexedItem[]> {
const filterPartition = await this.getIndexPartition(tenant, sortProperty);
const items: IndexedItem[] = [];
for await (const [ _key, value ] of filterPartition.iterator()) {
items.push(JSON.parse(value) as IndexedItem);
}
return items;
}

/**
* Returns items that match the exact property and value.
*/
Expand Down Expand Up @@ -540,10 +562,21 @@ export class IndexLevel {
}
}


private static shouldQueryWithInMemoryPaging(filters: Filter[], queryOptions: QueryOptions): boolean {
// if there is a specific recordId in any of the filters, return true immediately.
if (filters.find(({ recordId }) => recordId !== undefined) !== undefined) {
for (const filter of filters) {
if (!IndexLevel.isFilterConcise(filter, queryOptions)) {
return false;
}
}

// only use in-memory paging if all filters are concise
return true;
}


private static isFilterConcise(filter: Filter, queryOptions: QueryOptions): boolean {
// if there is a specific recordId in the filter, return true immediately.
if (filter.recordId !== undefined) {
return true;
}

Expand All @@ -552,16 +585,25 @@ export class IndexLevel {
return false;
}

// if contextId, protocolPath or schema are specified in any of the filter properties and no cursor exists, we use in-memory paging
if (filters.find(({ contextId, protocol, protocolPath, schema }) => {
return contextId !== undefined ||
protocol !== undefined ||
protocolPath !== undefined ||
schema !== undefined;
}) !== undefined) {
// NOTE: remaining conditions will have cursor

if (filter.contextId !== undefined) {
return true;
}

if (filter.protocol !== undefined || filter.protocolPath !== undefined) {
return true;
}

if (filter.protocol !== undefined || filter.parentId === undefined) {
return true;
}

if (filter.protocol === undefined && filter.schema !== undefined) {
return true;
}

// all else
return false;
}
}
169 changes: 37 additions & 132 deletions src/utils/filter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export class FilterUtility {
* @param filter
* @returns true if all of the filter properties match.
*/
private static matchFilter(indexedValues: KeyValues, filter: Filter): boolean {
public static matchFilter(indexedValues: KeyValues, filter: Filter): boolean {
// set of unique query properties.
// if count of missing property matches is 0, it means the data/object fully matches the filter
const missingPropertyMatches: Set<string> = new Set([ ...Object.keys(filter) ]);
Expand Down Expand Up @@ -155,156 +155,61 @@ export class FilterUtility {
export class FilterSelector {

/**
* Reduces an array of incoming Filters into an array of more efficient filters
* The length of the returned Filters array is always less than or equal to that of the input filters array.
* Reduce Filter so that it is a filter that can be quickly executed against the DB.
*/
static reduceFilters(filters: Filter[]): Filter[] {

// we extract any recordId filters and the remaining filters which do not have a recordId property
const { idFilters, remainingFilters } = this.extractIdFilters(filters);
// if there are no remaining filters, we only query by the idFilters
if (remainingFilters.length === 0) {
return idFilters;
static reduceFilter(filter: Filter): Filter {
// if there is only one or no property, we have no way to reduce it further
const filterProperties = Object.keys(filter);
if (filterProperties.length <= 1) {
return filter;
}

const commonFilter = this.extractCommonFilter(remainingFilters);
if (commonFilter !== undefined) {
return [ ...idFilters, commonFilter ];
}
// else there is are least 2 filter properties, since zero property is not allowed

// extract any range filters from the remaining filters
const { rangeFilters, remainingFilters: remainingAfterRange } = this.extractRangeFilters(remainingFilters);
// if all of there are no remaining filters we return the RangeFilters along with any idFilters.
if (remainingAfterRange.length === 0) {
return [ ...idFilters, ...rangeFilters ];
}
const { recordId, attester, parentId, recipient, contextId, author, protocolPath, schema, protocol, ...remainingProperties } = filter;

const commonAfterRange = this.extractCommonFilter(remainingAfterRange);
if (commonAfterRange !== undefined){
return [ ...idFilters, ...rangeFilters, commonAfterRange ];
if (recordId !== undefined) {
return { recordId };
}

const finalRemaining = remainingAfterRange.map(filter => {

const { contextId, schema, protocol, protocolPath, author, ...remaining } = filter;
if (contextId !== undefined && FilterUtility.isEqualFilter(contextId)) {
return { contextId };
} else if (schema !== undefined && FilterUtility.isEqualFilter(schema)) {
return { schema };
} else if (protocolPath !== undefined && FilterUtility.isEqualFilter(protocolPath)) {
return { protocolPath };
} else if (protocol !== undefined && FilterUtility.isEqualFilter(protocol)) {
return { protocol };
} else if (author !== undefined && FilterUtility.isEqualFilter(author)) {
return { author };
} else {

return this.getFirstFilterPropertyThatIsNotABooleanEqualFilter(remaining) || filter;
}
});

return [ ...idFilters, ...rangeFilters, ...finalRemaining ];
}

/**
* Extracts a single range filter from each of the input filters to return.
* Naively chooses the first range filter it finds, this could be improved.
*
* @returns an array of Filters with each filter containing a single RangeFilter property.
*/
private static extractRangeFilters(filters: Filter[]): { rangeFilters: Filter[], remainingFilters: Filter[] } {
const rangeFilters: Filter[] = [];
const remainingFilters: Filter[] = [];
for (const filter of filters) {
const filterKeys = Object.keys(filter);
const rangeFilterKey = filterKeys.find(filterProperty => FilterUtility.isRangeFilter(filter[filterProperty]));
if (rangeFilterKey === undefined) {
remainingFilters.push(filter);
continue;
}
const rangeFilter:Filter = {};
rangeFilter[rangeFilterKey] = filter[rangeFilterKey];
rangeFilters.push(rangeFilter);
if (attester !== undefined) {
return { attester };
}
return { rangeFilters, remainingFilters };
}

private static extractIdFilters(filters: Filter[]): { idFilters: Filter[], remainingFilters: Filter[] } {
const idFilters: Filter[] = [];
const remainingFilters: Filter[] = [];
for (const filter of filters) {
const { recordId } = filter;
// we determine if any of the filters contain a recordId property;
// we don't use range filters with these, so either Equality or OneOf filters should be used
if (recordId !== undefined && (FilterUtility.isEqualFilter(recordId) || FilterUtility.isOneOfFilter(recordId))) {
idFilters.push({ recordId });
continue;
}
remainingFilters.push(filter);
if (parentId !== undefined) {
return { parentId };
}

return { idFilters: idFilters, remainingFilters };
}

private static extractCommonFilter(filters: Filter[]): Filter | undefined {
const { schema, contextId, protocol, protocolPath, author, ...remaining } = this.commonEqualFilters(filters);
if (recipient !== undefined) {
return { recipient };
}

// if we match any of these, we add them to our search filters and return immediately
// the order we are checking/returning is the order of priority
if (contextId !== undefined && FilterUtility.isEqualFilter(contextId)) {
// a common contextId exists between all filters
if (contextId !== undefined) {
return { contextId };
} else if ( schema !== undefined && FilterUtility.isEqualFilter(schema)) {
// a common schema exists between all filters
return { schema };
} else if (protocolPath !== undefined && FilterUtility.isEqualFilter(protocolPath)) {
// a common protocol exists between all filters
return { protocolPath };
} else if (protocol !== undefined && FilterUtility.isEqualFilter(protocol)) {
// a common protocol exists between all filters
return { protocol };
} else if (author !== undefined && FilterUtility.isEqualFilter(author)) {
// a common author exists between all filters
return { author };
}

// return the first common filter that isn't listed in priority with a boolean common filter being last priority.
return this.getFirstFilterPropertyThatIsNotABooleanEqualFilter(remaining);
}
// assumes author is not the tenant
if (author !== undefined) {
return { author };
}

private static getFirstFilterPropertyThatIsNotABooleanEqualFilter(filter: Filter): Filter | undefined {
const filterProperties = Object.keys(filter);
if (protocolPath !== undefined) {
return { protocolPath };
}

// find the first EqualFilter that is not a boolean
const firstProperty = filterProperties.find(filterProperty => {
const filterValue = filter[filterProperty];
return filterValue !== undefined && FilterUtility.isEqualFilter(filterValue) && typeof filterValue !== 'boolean';
});
// if a non boolean filter exists, set it as the only filter property and return
if (firstProperty !== undefined) {
const singlePropertyFilter:Filter = {};
singlePropertyFilter[firstProperty] = filter[firstProperty];
return singlePropertyFilter;
if (schema !== undefined) {
return { schema };
}

return;
}
if (protocol !== undefined) {
return { protocol };
}

/**
* Given an array of filters, it returns a single filter with common EqualFilter per property.
* If there are no common filters, the returned filter is empty.
*/
private static commonEqualFilters(filters: Filter[]): Filter {
return filters.reduce((prev, current) => {
const filterCopy = { ...prev };
for (const property in filterCopy) {
const filterValue = filterCopy[property];
const compareValue = current[property];
if (!FilterUtility.isEqualFilter(filterValue) || !FilterUtility.isEqualFilter(compareValue) || filterValue !== compareValue) {
delete filterCopy[property];
}
}
return filterCopy;
});
// else just return whatever property, we can optimize further later
const remainingPropertyNames = Object.keys(remainingProperties);
const firstRemainingProperty = remainingPropertyNames[0];
const singlePropertyFilter: Filter = {};
singlePropertyFilter[firstRemainingProperty] = filter[firstRemainingProperty];
return singlePropertyFilter;
}
}
Loading

0 comments on commit 3b6e68e

Please sign in to comment.