update EventsFilter types, error handling, convertFilter/parse

src/core/dwn-error.ts

@@ -82,6 +82,7 @@ export enum DwnErrorCode {
   ProtocolsQueryUnauthorized = 'ProtocolsQueryUnauthorized',
   RecordsDecryptNoMatchingKeyEncryptedFound = 'RecordsDecryptNoMatchingKeyEncryptedFound',
   RecordsDeleteAuthorizationFailed = 'RecordsDeleteAuthorizationFailed',
+  RecordsFilterPublishedSortInvalid = 'RecordsFilterPublishedSortInvalid',
   RecordsGrantAuthorizationConditionPublicationProhibited = 'RecordsGrantAuthorizationConditionPublicationProhibited',
   RecordsGrantAuthorizationConditionPublicationRequired = 'RecordsGrantAuthorizationConditionPublicationRequired',
   RecordsGrantAuthorizationScopeContextIdMismatch = 'RecordsGrantAuthorizationScopeContextIdMismatch',

src/interfaces/events-query.ts

@@ -1,17 +1,20 @@
 import type { Filter } from '../types/query-types.js';
+import type { ProtocolsQueryFilter } from '../types/protocols-types.js';
 import type { Signer } from '../types/signer.js';
-import type { EventsFilter, EventsQueryDescriptor, EventsQueryMessage } from '../types/event-types.js';
+import type { EventsFilter, EventsQueryDescriptor, EventsQueryFilter, EventsQueryMessage, EventsRecordsFilter } from '../types/event-types.js';
 
 import { AbstractMessage } from '../core/abstract-message.js';
+import { FilterUtility } from '../utils/filter.js';
 import { Message } from '../core/message.js';
+import { ProtocolsQuery } from '../interfaces/protocols-query.js';
 import { Records } from '../utils/records.js';
 import { removeUndefinedProperties } from '../utils/object.js';
 import { Time } from '../utils/time.js';
 import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 
 export type EventsQueryOptions = {
   signer: Signer;
-  filters: EventsFilter[];
+  filters: EventsQueryFilter[];
   cursor?: string;
   messageTimestamp?: string;
 };
@@ -44,19 +47,74 @@ export class EventsQuery extends AbstractMessage<EventsQueryMessage>{
     return new EventsQuery(message);
   }
 
-  private static normalizeFilters(filters: EventsFilter[]): EventsFilter[] {
-    // currently all normalization filters are shared with `Records`.
-    return filters.map(filter => Records.normalizeFilter(filter));
+  private static normalizeFilters(filters: EventsQueryFilter[]): EventsQueryFilter[] {
+
+    const eventsQueryFilters: EventsQueryFilter[] = [];
+
+    // normalize each filter individually by the type of filter it is.
+    for (const filter of filters) {
+      if (this.isRecordsFilter(filter)) {
+        eventsQueryFilters.push(Records.normalizeFilter(filter));
+      } else if (this.isProtocolsFilter(filter)) {
+        const protocolFilter = ProtocolsQuery.normalizeFilter(filter);
+        eventsQueryFilters.push(protocolFilter!);
+      } else {
+        eventsQueryFilters.push(filter as EventsFilter);
+      }
+    }
+
+    return eventsQueryFilters;
   }
 
+
   /**
    *  Converts an incoming array of EventsFilter into a Filter usable by EventLog.
    *
    * @param filters An array of EventsFilter
    * @returns {Filter[]} an array of generic Filter able to be used when querying.
    */
-  public static convertFilters(filters: EventsFilter[]): Filter[] {
-    //currently only the range criterion for Records need to be converted
-    return filters.map(filter => Records.convertFilter(filter));
+  public static convertFilters(filters: EventsQueryFilter[]): Filter[] {
+
+    const eventsQueryFilters: Filter[] = [];
+
+    // normalize each filter individually by the type of filter it is.
+    for (const filter of filters) {
+      if (this.isRecordsFilter(filter)) {
+        eventsQueryFilters.push(Records.convertFilter(filter));
+      } else if (this.isProtocolsFilter(filter)) {
+        eventsQueryFilters.push({ ...filter });
+      } else {
+        eventsQueryFilters.push(this.convertFilter(filter));
+      }
+    }
+
+    return eventsQueryFilters;
+  }
+
+  private static convertFilter(filter: EventsFilter): Filter {
+    const filterCopy = { ...filter } as Filter;
+
+    const { messageTimestamp } = filter;
+    const messageTimestampFilter = messageTimestamp ? FilterUtility.convertRangeCriterion(messageTimestamp) : undefined;
+    if (messageTimestampFilter) {
+      filterCopy.messageTimestamp = messageTimestampFilter;
+    }
+
+    return filterCopy as Filter;
+  }
+
+  private static isProtocolsFilter(filter: EventsQueryFilter): filter is ProtocolsQueryFilter {
+    return 'protocol' in filter;
+  }
+
+  private static isRecordsFilter(filter: EventsQueryFilter): filter is EventsRecordsFilter {
+    return 'dateCreated' in filter ||
+      'dataFormat' in filter ||
+      'parentId' in filter ||
+      'recordId' in filter ||
+      'schema' in filter ||
+      'protocolPath' in filter || // explicitly ignore `protocol` as it will be handled by the protocol filter
+      'recipient' in filter;
   }
+
 }

src/interfaces/protocols-query.ts

@@ -65,7 +65,7 @@ export class ProtocolsQuery extends AbstractMessage<ProtocolsQueryMessage> {
     return protocolsQuery;
   }
 
-  private static normalizeFilter(filter: ProtocolsQueryFilter | undefined): ProtocolsQueryFilter | undefined {
+  static normalizeFilter(filter: ProtocolsQueryFilter | undefined): ProtocolsQueryFilter | undefined {
     if (filter === undefined) {
       return undefined;
     }

src/interfaces/records-query.ts

@@ -1,9 +1,10 @@
 import type { DelegatedGrantMessage } from '../types/delegated-grant-message.js';
 import type { Pagination } from '../types/message-types.js';
 import type { Signer } from '../types/signer.js';
-import type { DateSort, RecordsFilter, RecordsQueryDescriptor, RecordsQueryMessage } from '../types/records-types.js';
+import type { RecordsFilter, RecordsQueryDescriptor, RecordsQueryMessage } from '../types/records-types.js';
 
 import { AbstractMessage } from '../core/abstract-message.js';
+import { DateSort } from '../types/records-types.js';
 import { Message } from '../core/message.js';
 import { Records } from '../utils/records.js';
 import { removeUndefinedProperties } from '../utils/object.js';
@@ -47,6 +48,16 @@ export class RecordsQuery extends AbstractMessage<RecordsQueryMessage> {
         );
       }
     }
+
+    if (message.descriptor.filter.published === false) {
+      if (message.descriptor.dateSort === DateSort.PublishedAscending || message.descriptor.dateSort === DateSort.PublishedDescending) {
+        throw new DwnError(
+          DwnErrorCode.RecordsFilterPublishedSortInvalid,
+          `queries must not filter for \`published:false\` and sort by ${message.descriptor.dateSort}`
+        );
+      }
+    }
+
     if (message.descriptor.filter.protocol !== undefined) {
       validateProtocolUrlNormalized(message.descriptor.filter.protocol);
     }

src/store/index-level.ts

@@ -293,6 +293,8 @@ export class IndexLevel {
    *
    * @param matchFilters the filters passed to the parent query.
    * @param searchFilters the modified filters used for the LevelDB query to search for a subset of items to match against.
+   *
+   * @throws {DwnErrorCode.IndexLevelInMemoryInvalidSortProperty} if an invalid sort property is provided.
    */
   async queryWithInMemoryPaging(
     tenant: string,

src/types/event-types.ts

@@ -1,15 +1,21 @@
 import type { GenericMessageReply } from '../core/message-reply.js';
+import type { ProtocolsQueryFilter } from './protocols-types.js';
+import type { RangeCriterion } from './query-types.js';
 import type { RecordsFilter } from './records-types.js';
 import type { AuthorizationModel, GenericMessage } from './message-types.js';
 import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 
-export type EventsFilter = RecordsFilter & {
-  /** optional array of methods to filter */
-  method?: string[];
-  /** optional array of interfaces to filter */
-  interface?: string[];
+export type EventsFilter = {
+  method?: string;
+  interface?: string;
+  messageTimestamp?: RangeCriterion;
 };
 
+// We only allow filtering for events by immutable properties, the omitted properties could be different per subsequent writes.
+export type EventsRecordsFilter = Omit<RecordsFilter, 'author' | 'attester' | 'published' | 'dataSize' | 'dataCid' | 'datePublished' | 'dateUpdated' >;
+
+export type EventsQueryFilter = EventsFilter | EventsRecordsFilter | ProtocolsQueryFilter;
+
 export type EventsGetDescriptor = {
   interface : DwnInterfaceName.Events;
   method: DwnMethodName.Get;
@@ -30,7 +36,7 @@ export type EventsQueryDescriptor = {
   interface: DwnInterfaceName.Events;
   method: DwnMethodName.Query;
   messageTimestamp: string;
-  filters: EventsFilter[];
+  filters: EventsQueryFilter[];
   cursor?: string;
 };
 

src/types/query-types.ts

@@ -37,4 +37,16 @@ export type FilterValue = EqualFilter | OneOfFilter | RangeFilter;
 
 export type Filter = {
   [property: string]: FilterValue;
+};
+
+export type RangeCriterion = {
+  /**
+   * Inclusive starting date-time.
+   */
+  from?: string;
+
+  /**
+   * Inclusive end date-time.
+   */
+  to?: string;
 };

src/types/records-types.ts

@@ -3,10 +3,10 @@ import type { GeneralJws } from './jws-types.js';
 import type { GenericMessageReply } from '../core/message-reply.js';
 import type { KeyDerivationScheme } from '../utils/hd-key.js';
 import type { PublicJwk } from './jose-types.js';
-import type { RangeFilter } from './query-types.js';
 import type { Readable } from 'readable-stream';
 import type { AuthorizationModel, GenericMessage, GenericSignaturePayload, Pagination } from './message-types.js';
 import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
+import type { RangeCriterion, RangeFilter } from './query-types.js';
 
 export enum DateSort {
   CreatedAscending = 'createdAscending',
@@ -122,18 +122,6 @@ export type RecordsFilter = {
   dateUpdated?: RangeCriterion;
 };
 
-export type RangeCriterion = {
-  /**
-   * Inclusive starting date-time.
-   */
-  from?: string;
-
-  /**
-   * Inclusive end date-time.
-   */
-  to?: string;
-};
-
 export type RecordsWriteAttestationPayload = {
   descriptorCid: string;
 };

src/utils/filter.ts

@@ -1,4 +1,4 @@
-import type { EqualFilter, Filter, FilterValue, KeyValues, OneOfFilter, QueryOptions, RangeFilter } from '../types/query-types.js';
+import type { EqualFilter, Filter, FilterValue, KeyValues, OneOfFilter, QueryOptions, RangeCriterion, RangeFilter } from '../types/query-types.js';
 
 import { isEmptyObject } from './object.js';
 
@@ -135,6 +135,26 @@ export class FilterUtility {
     };
     return false;
   }
+
+  static convertRangeCriterion(inputFilter: RangeCriterion): RangeFilter | undefined {
+    let rangeFilter: RangeFilter | undefined;
+    if (inputFilter.to !== undefined && inputFilter.from !== undefined) {
+      rangeFilter = {
+        gte : inputFilter.from,
+        lt  : inputFilter.to,
+      };
+    } else if (inputFilter.to !== undefined) {
+      rangeFilter = {
+        lt: inputFilter.to,
+      };
+    } else if (inputFilter.from !== undefined) {
+      rangeFilter = {
+        gte: inputFilter.from,
+      };
+    }
+    return rangeFilter;
+  }
+
 }
 
 export class FilterSelector {
@@ -158,7 +178,6 @@ export class FilterSelector {
       });
   }
 
-  //TODO: return a single filter, this may have to change where/how this method is used.
   private static checkForIdSearches(filters: Filter[]): { searchFilters: Filter[], remainingFilters: Filter[] } {
     const searchFilters: Filter[] = [];
     const remainingFilters: Filter[] = [];

src/utils/records.ts

@@ -1,14 +1,16 @@
 import type { DerivedPrivateJwk } from './hd-key.js';
+import type { Filter } from '../types/query-types.js';
 import type { GenericSignaturePayload } from '../types/message-types.js';
 import type { Readable } from 'readable-stream';
-import type { Filter, RangeFilter } from '../types/query-types.js';
-import type { RangeCriterion, RecordsDeleteMessage, RecordsFilter, RecordsQueryMessage, RecordsReadMessage, RecordsWriteDescriptor, RecordsWriteMessage } from '../types/records-types.js';
+import type { RecordsDeleteMessage, RecordsFilter, RecordsQueryMessage, RecordsReadMessage, RecordsWriteDescriptor, RecordsWriteMessage } from '../types/records-types.js';
 
 import { DateSort } from '../types/records-types.js';
 import { Encoder } from './encoder.js';
 import { Encryption } from './encryption.js';
+import { FilterUtility } from './filter.js';
 import { KeyDerivationScheme } from './hd-key.js';
 import { Message } from '../core/message.js';
+import { removeUndefinedProperties } from './object.js';
 import { Secp256k1 } from './secp256k1.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
 import { normalizeProtocolUrl, normalizeSchemaUrl } from './url.js';
@@ -236,11 +238,14 @@ export class Records {
       schema = normalizeSchemaUrl(filter.schema);
     }
 
-    return {
+    const filterCopy = {
       ...filter,
       protocol,
       schema,
     };
+
+    removeUndefinedProperties(filterCopy);
+    return filterCopy;
   }
 
   /**
@@ -253,50 +258,31 @@ export class Records {
     const filterCopy = { ...filter } as Filter;
 
     const { dateCreated, datePublished, dateUpdated } = filter;
-    const dateCreatedFilter = dateCreated ? this.convertRangeCriterion(dateCreated) : undefined;
+    const dateCreatedFilter = dateCreated ? FilterUtility.convertRangeCriterion(dateCreated) : undefined;
     if (dateCreatedFilter) {
       filterCopy.dateCreated = dateCreatedFilter;
     }
 
-    const datePublishedFilter = datePublished ? this.convertRangeCriterion(datePublished): undefined;
+    const datePublishedFilter = datePublished ? FilterUtility.convertRangeCriterion(datePublished): undefined;
     if (datePublishedFilter) {
       // only return published records when filtering with a datePublished range.
       filterCopy.published = true;
       filterCopy.datePublished = datePublishedFilter;
     }
 
-    //if sorting by published, results must filter for published
-    if (dateSort === DateSort.PublishedAscending || dateSort === DateSort.PublishedDescending) {
+    // if we sort by `PublishedAscending` or `PublishedDescending` we must filter for only published records.
+    if (filterCopy.published !== true && (dateSort === DateSort.PublishedAscending || dateSort === DateSort.PublishedDescending)) {
       filterCopy.published = true;
     }
 
-    const messageTimestampFilter = dateUpdated ? this.convertRangeCriterion(dateUpdated) : undefined;
+    const messageTimestampFilter = dateUpdated ? FilterUtility.convertRangeCriterion(dateUpdated) : undefined;
     if (messageTimestampFilter) {
       filterCopy.messageTimestamp = messageTimestampFilter;
       delete filterCopy.dateUpdated;
     }
     return filterCopy as Filter;
   }
 
-  private static convertRangeCriterion(inputFilter: RangeCriterion): RangeFilter | undefined {
-    let rangeFilter: RangeFilter | undefined;
-    if (inputFilter.to !== undefined && inputFilter.from !== undefined) {
-      rangeFilter = {
-        gte : inputFilter.from,
-        lt  : inputFilter.to,
-      };
-    } else if (inputFilter.to !== undefined) {
-      rangeFilter = {
-        lt: inputFilter.to,
-      };
-    } else if (inputFilter.from !== undefined) {
-      rangeFilter = {
-        gte: inputFilter.from,
-      };
-    }
-    return rangeFilter;
-  }
-
   /**
    * Validates the referential integrity regarding delegated grant.
    * @param signaturePayload Decoded payload of the signature of the message. `undefined` if message is not signed.