Update SRP sync to use ESI rather than ZKillboard

Implements a new version of syncKillmails that employs flows and
updated market data sources.

Attempts to recreate the metadata provided by zkill as much as possible,
namely the "npc" and fitted/totalValue fields, but we can't recreate
everything, sadly (especially the "solo" field).

Additionally, the ESI endpoint only returns killmails for which a corp
member provided the killing blow, so we will lose access to some kills
that were previously provided by ZKill. however, the SRP system really
only needs access to the losses; the kills exist solely to provide
context for SRP triagers.

A couple of additional pieces of work need to occur after this change.
First, we need to rename all usages of ZKillmail to something more
generic like AnnotatedKillmail. Second, many of our losses do not
appear in ZKill anymore, so we need to provide a detail view page
for every killmail we store.

schema/021-cascade-killmail-deletions.cjs

@@ -0,0 +1,42 @@
+/**
+ * Adds ON CASCADE DELETE to the killmail_id part of killmailBattle
+ */
+
+exports.up = async function (trx) {
+  await trx.schema.alterTable("killmailBattle", (table) => {
+    table.dropForeign("killmail");
+  });
+  await trx.schema.alterTable("killmailBattle", (table) => {
+    table.foreign("killmail").references("killmail.id").onDelete("CASCADE");
+  });
+
+  await trx.schema.alterTable("srpVerdict", (table) => {
+    table.dropForeign("killmail");
+    table.dropForeign("reimbursement");
+  });
+  await trx.schema.alterTable("srpVerdict", (table) => {
+    table.foreign("killmail").references("killmail.id").onDelete("CASCADE");
+    table
+      .foreign("reimbursement")
+      .references("srpReimbursement.id")
+      .onDelete("SET NULL");
+  });
+};
+
+exports.down = async function (trx) {
+  await trx.schema.alterTable("killmailBattle", (table) => {
+    table.dropForeign("killmail");
+  });
+  await trx.schema.alterTable("killmailBattle", (table) => {
+    table.foreign("killmail").references("killmail.id");
+  });
+
+  await trx.schema.alterTable("srpVerdict", (table) => {
+    table.dropForeign("killmail");
+    table.dropForeign("reimbursement");
+  });
+  await trx.schema.alterTable("srpVerdict", (table) => {
+    table.foreign("killmail").references("killmail.id");
+    table.foreign("reimbursement").references("srpReimbursement.id");
+  });
+};

src/server/data-source/esi/EsiEntity.ts

@@ -2,6 +2,7 @@ import util, { InspectOptions } from "node:util";
 
 export class EsiEntity {
   constructor(
+    public readonly category: "character" | "corporation" | "type",
     public readonly id: number,
     public readonly name: string,
   ) {}
@@ -20,9 +21,13 @@ export class EsiEntity {
 }
 
 export function esiChar(id: number, name: string) {
-  return new EsiEntity(id, name);
+  return new EsiEntity("character", id, name);
 }
 
 export function esiCorp(id: number, name: string) {
-  return new EsiEntity(id, name);
+  return new EsiEntity("corporation", id, name);
+}
+
+export function esiType(id: number, name: string) {
+  return new EsiEntity("type", id, name);
 }

src/server/data-source/esi/EsiKillmail.ts

@@ -20,10 +20,11 @@ export interface EsiKillmail {
  * associated with players; sometimes they aren't.
  *
  * Below are a few common patterns, but in general your code must expect that
- * ANY combination of the optional fields below can occur. Of particular note
- * is that there is no reliable way to tell the difference between a
- * player corporation-owned structure and a "corporation NPC" (beyond checking
- * the corporation ID against a list of known NPC corps).
+ * ANY combination of the optional fields below can occur. Note that CCP has
+ * reworked how NPC ships are represented; these used to be primarily associated
+ * with a faction_id. Now, many NPC ships have an associated corporation and
+ * faction_id has been repurposed to (sometimes) represent faction warfare
+ * alignment (and can be present on real players as a result).
  *
  * Finally, a few oddities. The ship_type_id property is present for most
  * entities but is sometimes left blank for reasons known only to CCP. If an
@@ -40,6 +41,7 @@ export interface EsiKillmail {
  *  character_id,
  *  corpororation_id,
  *  alliance_id?,
+ *  faction_id?
  * }
  *
  * Corporation-owned structure
@@ -48,22 +50,27 @@ export interface EsiKillmail {
  *  alliance_id?,
  * }
  *
- * Normal NPC
+ * Normal NPC (pirates, CONCORD, faction police, etc.)
  * {
- *  faction_id,
+ *  corporation_id,
  * }
  *
  * "Character" NPC (Arithmos Tyrranos)
  * {
  *  character_id,
  *  corporation_id,
+ * }
+ *
+ * Old school NPCs (these are far less common but still exist)
+ * {
  *  faction_id,
  * }
  *
- * "Corporation" NPC (CONCORD, Faction Police, etc.)
+ * Environment effects
  * {
- *  corporation_id,
+ *  faction_id, // This is often (always?) the faction 500021 ("Unknown")
  * }
+ *
  */
 
 export interface Victim {

src/server/data-source/esi/EsiScope.ts

@@ -1,5 +1,6 @@
 export enum EsiScope {
   OPEN_WINDOW = "esi-ui.open_window.v1",
+  CORP_READ_KILLMAILS = "esi-killmails.read_corporation_killmails.v1",
   CORP_READ_MEMBERSHIP = "esi-corporations.read_corporation_membership.v1",
   CORP_READ_TITLES = "esi-corporations.read_titles.v1",
   CORP_TRACK_MEMBERS = "esi-corporations.track_members.v1",

src/server/data-source/esi/flow/paginatedEsiEndpoint.ts

@@ -0,0 +1,60 @@
+import { flow } from "../../../util/flow/flow.js";
+import { EsiEndpoint } from "../EsiEndpoint.js";
+import { EsiEndpointParams } from "../fetch/EsiEndpointParams.js";
+import { fetchEsiEx } from "../fetch/fetchEsi.js";
+
+/**
+ * Exposes a paginated ESI endpoint as a flow source
+ *
+ * Provided an ESI endpoint that is paginated (i.e. returns an array of data
+ * split across multiple pages), creates a flow that emits elements in order
+ * from those pages, advancing pages when necessary.
+ *
+ * Respects backpressure, so new pages are only fetched when old ones are
+ * completely consumed.
+ */
+export function paginatedEsiEndpoint<T extends ArrayEsiEndpoint>(
+  endpoint: T,
+  params: Omit<EsiEndpointParams<T>, "page">,
+  maxAttempts = 2,
+) {
+  return flow.defineSource<EndpointRowItem<T>>((node) => {
+    let nextPage = 1;
+    let maxPages = 1;
+
+    const activeParams = Object.assign({ page: nextPage }, params);
+
+    return {
+      async onRead() {
+        if (nextPage > maxPages) {
+          node.close();
+          return;
+        }
+        activeParams.page = nextPage;
+
+        const pageResult = await fetchEsiEx(
+          endpoint,
+          activeParams as EsiEndpointParams<T>,
+          maxAttempts,
+        );
+
+        maxPages = pageResult.pageCount;
+        const items = pageResult.data;
+        for (const item of items) {
+          node.emit(item);
+        }
+
+        nextPage++;
+      },
+    };
+  });
+}
+
+interface ArrayEsiEndpoint extends EsiEndpoint {
+  response: unknown[];
+  query: {
+    page: number;
+  };
+}
+
+type EndpointRowItem<T extends ArrayEsiEndpoint> = T["response"][0];

src/server/data-source/zkillboard/ZKillmail.ts

@@ -1,3 +1,4 @@
+import { SimpleNumMap } from "../../../shared/util/simpleTypes.js";
 import { EsiKillmail } from "../esi/EsiKillmail.js";
 
 /**
@@ -15,13 +16,14 @@ export type ZKillmail = EsiKillmail & ZKillDescriptor;
 export interface ZKillDescriptor {
   killmail_id: number;
   zkb: {
-    locationID: number;
+    locationID?: number;
     hash: string;
     fittedValue: number;
     totalValue: number;
-    points: number;
+    prices?: SimpleNumMap<number>;
+    points?: number;
     npc: boolean;
-    solo: boolean;
-    awox: boolean;
+    solo?: boolean;
+    awox?: boolean;
   };
 }

src/server/db/tnex/Query.ts

@@ -8,11 +8,21 @@ export class Query<
   T extends object,
   R /* return type */,
 > extends FilterableQuery<T> {
+  private _printQuery = false;
+
   constructor(scoper: Scoper, query: Knex.QueryBuilder) {
     super(scoper, query);
   }
 
   public run(): Promise<R> {
+    if (this._printQuery) {
+      console.log(this._query.toString());
+    }
     return this._query as Promise<any>;
   }
+
+  public printQuery(): this {
+    this._printQuery = true;
+    return this;
+  }
 }

src/server/domain/battle/createPendingBattles.ts

@@ -47,6 +47,10 @@ export async function createPendingBattles(db: Tnex, logger: Logger) {
     await pipelinePr(reader, creator, writer);
 
     logger.info(`Created ${writer.getNewBattleCount()} new battles.`);
+
+    return {
+      newBattleCount: writer.getNewBattleCount(),
+    };
   });
 }
 

src/server/infra/logging/Logger.ts

@@ -2,13 +2,13 @@
  * Generic interface for logging.
  */
 export interface Logger {
-  crit(message: string, error?: Error, data?: object): void;
-  error(message: string, error?: Error, data?: object): void;
-  warn(message: string, error?: Error, data?: object): void;
+  crit(message: string, error?: unknown, data?: object): void;
+  error(message: string, error?: unknown, data?: object): void;
+  warn(message: string, error?: unknown, data?: object): void;
   info(message: string, data?: object): void;
   verbose(message: string, data?: object): void;
   debug(message: string, data?: object): void;
-  log(level: LogLevel, message: string, error?: Error, data?: object): void;
+  log(level: LogLevel, message: string, error?: unknown, data?: object): void;
 }
 
 export enum LogLevel {

src/server/infra/logging/WitnessLogger.ts

@@ -10,15 +10,15 @@ export class WitnessLogger implements Logger {
     this._tag = tag;
   }
 
-  crit(message: string, error?: Error, data?: object): void {
+  crit(message: string, error?: unknown, data?: object): void {
     this.log(LogLevel.CRIT, message, error, data);
   }
 
-  error(message: string, error?: Error, data?: object): void {
+  error(message: string, error?: unknown, data?: object): void {
     this.log(LogLevel.ERROR, message, error, data);
   }
 
-  warn(message: string, error?: Error, data?: object): void {
+  warn(message: string, error?: unknown, data?: object): void {
     this.log(LogLevel.WARN, message, error, data);
   }
 
@@ -34,7 +34,7 @@ export class WitnessLogger implements Logger {
     this.log(LogLevel.DEBUG, message, undefined, data);
   }
 
-  log(level: LogLevel, message: string, error?: Error, data?: object): void {
+  log(level: LogLevel, message: string, error?: unknown, data?: object): void {
     const levelTag = logLevelToProtocolTag(level);
     logMessage(levelTag, this._formatMessage(message, data));
     if (error) {