Open evidence and Wiki scraper

.env.sample

@@ -1,9 +1,2 @@
-API_PREFIX=/v1
-
-SOLR_PORT=<MY_SOLR_PORT>
-SOLR_CORE=<MY_SOLR_CORE>
-SOLR_HOST=<MY_SOLR_HOST>
-
-MONGO_DEV_CONN_URL=mongodb://<MY_MONGO_INSTANCE>
-MONGO_LOCAL_CONN_URL=mongodb://<MY_MONGO_INSTANCE>
-MONGO_DB_NAME=<MY_DB_NAME>
+DATABASE_URL=postgresql://username:password@localhost:5432/debate-cards
+DOCUMENT_PATH=./documents

.gitignore

@@ -15,4 +15,5 @@ prisma/migrations
 !.yarn/sdks
 !.yarn/versions
 *.docx
-*.html
+*.html
+tmp

package.json

@@ -11,19 +11,16 @@
   },
   "dependencies": {
     "@prisma/client": "^3.8.1",
+    "axios": "^0.25.0",
     "cheerio": "1.0.0-rc.3",
     "docx": "^6.0.3",
     "dotenv": "^6.2.0",
     "htmlparser2": "^7.2.0",
     "lodash": "^4.17.15",
-    "mammoth": "^1.4.19",
-    "node-pandoc-promise": "^0.0.6",
-    "p-ratelimit": "^1.0.1",
     "redis": "^4.0.4",
-    "sqlite3": "^5.0.0",
-    "tmp-promise": "^3.0.2",
     "typescript-collections": "^1.3.3",
-    "unzipper": "^0.10.11"
+    "unzipper": "^0.10.11",
+    "zod": "^3.17.3"
   },
   "devDependencies": {
     "@types/cheerio": "^0.22.21",
@@ -50,7 +47,8 @@
     "prettier": "^2.0.5",
     "prisma": "^3.8.1",
     "ts-node-dev": "^1.0.0-pre.50",
+    "tsconfig-paths": "^3.12.0",
     "tscpaths": "^0.0.9",
     "typescript": "^4.5.2"
   }
-}
+}

prisma/schema.prisma

@@ -48,6 +48,32 @@ model Evidence {
   root EvidenceBucket? @relation("root")
 }
 
+model Round {
+  id        Int        @id @default(autoincrement())
+  createdAt DateTime   @default(now())
+  updatedAt DateTime   @updatedAt
+  gid       String     @unique
+  status    FileStatus @default(PENDING)
+
+  wiki   String
+  school String
+  team   String
+  side   Side
+
+  entryDate  DateTime
+  tournament String
+  roundNum   String
+  opponent   String
+  judge      String
+
+  roundReport String?
+  cites       String[]
+
+  openSourceUrl String?
+  openSource    File?   @relation(fields: [openSourceId], references: [gid])
+  openSourceId  String?
+}
+
 model File {
   id        Int      @id @default(autoincrement())
   createdAt DateTime @default(now())
@@ -63,19 +89,25 @@ model File {
 
   evidence Evidence[]
   tags     Tags[]
+  Rounds   Round[]
 }
 
 model EvidenceSet {
   id        Int      @id @default(autoincrement())
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
-  name  String
+  name  String @unique
   label String
 
   files File[]
 }
 
+enum Side {
+  AFF
+  NEG
+}
+
 enum FileStatus {
   PENDING
   PROCESSING

src/actions/addFile.ts

@@ -9,11 +9,10 @@ export type FileData = Omit<Prisma.FileCreateInput, ExcludedFileFields>;
 
 export const onAddFile = new TypedEvent<{ gid: string }>();
 
-export default async (data: FileData): Promise<any> => {
+export default async (data: FileData): Promise<string> => {
   const buffer = await readFile(data.path);
 
   const { fileId: gid } = await makeId(buffer);
-  console.log(data.name, Date.now());
   const doc = await db.file.upsert({
     where: {
       gid,
@@ -32,5 +31,5 @@ export default async (data: FileData): Promise<any> => {
 
   onAddFile.emit({ gid: doc.gid });
 
-  return;
+  return doc.gid;
 };

src/actions/addRound.ts

@@ -0,0 +1,28 @@
+import { db, TypedEvent } from 'app/lib';
+import { loadRound, RoundInfo } from 'app/lib/debate-tools/wiki';
+import path from 'path';
+import { DownloadInfo } from './downloadFile';
+
+export const onAddRound = new TypedEvent<DownloadInfo>();
+export default async ({ url, roundId, gid }: RoundInfo): Promise<{ gid: string }> => {
+  const existing = await db.round.findUnique({ where: { gid } });
+  if (existing) return existing;
+
+  const data = { gid, ...(await loadRound(url, roundId)) };
+  if (data.status === 'PENDING') {
+    const { wiki, school, team, side, tournament, roundNum } = data;
+    onAddRound.emit({
+      url: data.openSourceUrl,
+      filePath: path.join(process.env.DOCUMENT_PATH, wiki, school, team, side, `${tournament}-Round${roundNum}.docx`),
+      evidenceSet: wiki,
+      roundGid: gid,
+    });
+  }
+
+  return db.round.upsert({
+    where: { gid },
+    create: data,
+    update: data,
+    select: { gid: true },
+  });
+};

src/actions/dedupeFile.ts

@@ -3,28 +3,24 @@ import { findParent } from 'app/lib/debate-tools';
 const updateLock: Record<number, Lock> = {};
 
 export default async ({ gid }: { gid: string }): Promise<any> => {
-  try {
-    const { id, fulltext } = await db.evidence.findUnique({ where: { gid }, select: { id: true, fulltext: true } });
-    const { updates, parent } = await findParent(id, fulltext);
+  const { id, fulltext } = await db.evidence.findUnique({ where: { gid }, select: { id: true, fulltext: true } });
+  const { updates, parent } = await findParent(id, fulltext);
 
-    // Only wait if it actually exists, otherwise it wont get set in time
-    if (updateLock[parent]) await updateLock[parent].promise;
-    const lock = (updateLock[parent] = new Lock());
+  // Only wait if it actually exists, otherwise it wont get set in time
+  if (updateLock[parent]) await updateLock[parent].promise;
+  const lock = (updateLock[parent] = new Lock());
 
-    Children.set(parent, updates);
-    const bucket = await db.evidenceBucket.upsert({
-      where: { rootId: parent },
-      create: { rootId: parent },
-      update: { count: { increment: updates.length } },
-    });
-    await db.evidence.updateMany({
-      where: { id: { in: updates.map(Number) } },
-      data: { bucketId: bucket.id },
-    });
+  Children.set(parent, updates);
+  const bucket = await db.evidenceBucket.upsert({
+    where: { rootId: parent },
+    create: { rootId: parent },
+    update: { count: { increment: updates.length } },
+  });
+  await db.evidence.updateMany({
+    where: { id: { in: updates.map(Number) } },
+    data: { bucketId: bucket.id },
+  });
 
-    lock.unlock();
-    if (updateLock[parent] === lock) delete updateLock[parent];
-  } catch (e) {
-    console.error(e);
-  }
+  lock.unlock();
+  if (updateLock[parent] === lock) delete updateLock[parent];
 };

src/actions/downloadFile.ts

@@ -0,0 +1,32 @@
+import { db } from 'app/lib/db';
+import { wikiDownload } from 'app/lib/request';
+import addFile from 'app/actions/addFile';
+import path from 'path';
+
+export interface DownloadInfo {
+  url: string;
+  filePath: string;
+  evidenceSet: string;
+  roundGid?: string;
+}
+
+export default async ({ url, filePath, evidenceSet, roundGid }: DownloadInfo): Promise<void> => {
+  try {
+    if (!url) return;
+    const result = await wikiDownload(url, filePath);
+    if ('err' in result) throw new Error(`Failed to download file ${url}: ${result.err.message}`);
+    const fileId = await addFile({
+      name: path.basename(filePath),
+      path: filePath,
+      evidenceSet: { connect: { name: evidenceSet } },
+    });
+    if (roundGid)
+      await db.round.update({
+        where: { gid: roundGid },
+        data: { openSource: { connect: { gid: fileId } }, status: 'PROCESSED' },
+      });
+  } catch (e) {
+    if (roundGid) await db.round.update({ where: { gid: roundGid }, data: { status: 'ERROR' } });
+    throw new Error(`Error downloading ${url}: ${e.message}`);
+  }
+};

src/actions/parseFile.ts

@@ -3,7 +3,7 @@ import { db, pipe } from 'app/lib';
 
 import { documentToTokens, extractCards, makeChildId } from 'app/lib/debate-tools';
 
-export default async ({ gid }: { gid: string }) => {
+export default async ({ gid }: { gid: string }): Promise<void> => {
   try {
     const cards = await pipe(
       (gid: string) => db.file.findUnique({ where: { gid }, select: { path: true } }),
@@ -16,7 +16,7 @@ export default async ({ gid }: { gid: string }) => {
     for (const card of cards) await addEvidence({ ...card, gid: makeChildId(gid, card.index), file: { gid } });
     await db.file.update({ where: { gid }, data: { status: 'PROCESSED' } });
   } catch (e) {
-    console.error(e);
     await db.file.update({ where: { gid }, data: { status: 'ERROR' } });
+    throw new Error(`Error parsing ${gid}: ${e.message}`);
   }
 };

src/constants/index.ts

@@ -1,3 +1,5 @@
+export * from './wiki';
+
 // Max number of files being parsed concurrently, allows parsing to take place while waiting for database response
 export const CONCURRENT_PARSERS = 10;
 // Max number of cards being deduplicated concurrently

src/constants/wiki.ts

@@ -0,0 +1,126 @@
+import { z } from 'zod';
+
+// Regex to get year from wiki name like hspolicy21
+export const WIKI_NAME_REGEX = /^(?<type>[a-z]+)(?<year>\d+)?$/;
+export const WIKITYPES = {
+  hspolicy: 'High School Policy',
+  hsld: 'High School LD',
+  hspf: 'High School PF',
+  opencaselist: 'College Policy',
+  openev: 'Open Evidence',
+  nfald: 'College LD',
+} as const;
+
+/*
+  Types for wiki api
+  Base Xwiki schema defined here https://github.com/xwiki/xwiki-platform/blob/master/xwiki-platform-core/xwiki-platform-rest/xwiki-platform-rest-model/src/main/resources/xwiki.rest.model.xsd
+*/
+const links = z.array(
+  z.object({
+    href: z.string().url(),
+    rel: z.string(),
+    type: z.string().nullable(),
+    hrefLang: z.null().nullable(),
+  }),
+);
+const summary = {
+  links,
+  id: z.string(),
+  guid: z.string(),
+  pageId: z.string(),
+  pageVersion: z.string(),
+  wiki: z.string(),
+  space: z.string(),
+  pageName: z.string(),
+  pageAuthor: z.string(),
+  pageAuthorName: z.string().nullable(),
+  className: z.string(),
+  number: z.number(),
+  headline: z.string().nullable(),
+};
+
+export const OBJECT_SUMMARIES = z.object({
+  links,
+  objectSummaries: z.array(z.object(summary)),
+});
+
+export const SPACES = z.object({
+  links,
+  spaces: z.array(
+    z.object({
+      links,
+      id: z.string(),
+      wiki: z.string(),
+      name: z.string(),
+      home: z.string(),
+      xwikiRelativeUrl: z.string(),
+      xwikiAbsoluteUrl: z.string().url(),
+    }),
+  ),
+});
+
+export const WIKIS = z.object({
+  links,
+  wikis: z.array(
+    z.object({
+      links,
+      id: z.string(),
+      name: z.string(),
+      description: z.string().nullable(),
+      owner: z.string().nullable(),
+    }),
+  ),
+});
+
+export const ATTACHMENTS = z.object({
+  links,
+  attachments: z.array(
+    z.object({
+      links,
+      id: z.string(),
+      name: z.string(),
+      size: z.number(),
+      longSize: z.number(),
+      version: z.string(),
+      pageId: z.string(),
+      pageVersion: z.string(),
+      mimeType: z.string(),
+      author: z.string(),
+      authorName: z.string().nullable(),
+      date: z.number(),
+      xwikiRelativeUrl: z.string(),
+      xwikiAbsoluteUrl: z.string().url(),
+      hierarchy: z.object({
+        items: z.array(z.object({ label: z.string(), name: z.string(), type: z.string(), url: z.string().url() })),
+      }),
+    }),
+  ),
+});
+
+const property = <T extends string>(name: T) =>
+  z.object({
+    links,
+    name: z.literal(name),
+    value: z.string(),
+    type: z.string(),
+  });
+
+// Info about specific objects found by querying https://openev.debatecoaches.org/rest/wikis/hspolicy21/classes/classname
+export const ROUND = z.object({
+  ...summary,
+  // Using .map messes up types
+  properties: z.tuple([
+    property('Cites'),
+    property('EntryDate'),
+    property('Judge'),
+    property('OpenSource'),
+    property('Opponent'),
+    property('Round'),
+    property('RoundID'),
+    property('RoundReport'),
+    property('Tags'),
+    property('Tournament'),
+    property('Video'),
+  ]),
+});
+export const CITE = property('Cites');