diff --git a/.vscode/launch.json b/.vscode/launch.json index 15c0decd..55c072e2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,7 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { "name": "frontend", "type": "node", @@ -16,6 +17,18 @@ ], "console": "integratedTerminal" }, + { + "name": "frontend build", + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/webapp/frontend", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "run", + "build" + ], + "console": "integratedTerminal" + }, { "name": "django shell", "type": "debugpy", @@ -56,6 +69,16 @@ ], "django": true, "program": "${workspaceFolder}/webapp/api/manage.py" + }, + { + "name": "process tasks", + "type": "debugpy", + "request": "launch", + "args": [ + "process_tasks" + ], + "django": true, + "program": "${workspaceFolder}/webapp/api/manage.py" } ] } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f3b6d695..3b760f85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,4 +82,3 @@ volumes: api-db: api-db-backup: solr-data: - diff --git a/webapp/api/api/metrics.py b/webapp/api/api/metrics.py index 918f71da..2a87a658 100644 --- a/webapp/api/api/metrics.py +++ b/webapp/api/api/metrics.py @@ -24,7 +24,7 @@ from torch import nn from api.admin import retrieve_project_data -from api.models import ProjectAnnotateEntities, ProjectMetrics as AppProjectMetrics +from api.models import AnnotatedEntity, ProjectAnnotateEntities, ProjectMetrics as AppProjectMetrics from core.settings import MEDIA_ROOT _dt_fmt = '%Y-%m-%d %H:%M:%S.%f' @@ -76,16 +76,20 @@ def __init__(self, mct_export_data: dict, cat: CAT): """ self.mct_export = mct_export_data self.cat = cat - self.project_names = [] - self.document_names = [] + self.projects2names = {} + self.projects2doc_ids = {} + self.docs2names = {} + self.docs2texts = {} self.annotations = self._annotations() def _annotations(self): ann_lst = [] for proj in self.mct_export['projects']: - self.project_names.append(proj) + self.projects2names[proj['id']] = proj['name'] + self.projects2doc_ids[proj['id']] = [doc['id'] for doc in proj['documents']] for doc in proj['documents']: - self.document_names.append(doc['name']) + self.docs2names[doc['id']] = doc['name'] + self.docs2texts[doc['id']] = doc['text'] for anns in doc['annotations']: meta_anns_dict = dict() for meta_ann in anns['meta_anns'].items(): @@ -135,6 +139,8 @@ def concept_summary(self, extra_cui_filter=None): fps, fns, tps, cui_prec, cui_rec, cui_f1, cui_counts, examples = self.cat._print_stats(data=self.mct_export, use_project_filters=True, extra_cui_filter=extra_cui_filter) + # remap tps, fns, fps to specific user annotations + examples = self.enrich_medcat_metrics(examples) concept_count_df['fps'] = concept_count_df['cui'].map(fps) concept_count_df['fns'] = concept_count_df['cui'].map(fns) concept_count_df['tps'] = concept_count_df['cui'].map(tps) @@ -154,6 +160,33 @@ def concept_summary(self, extra_cui_filter=None): concept_summary = [{k: list(v) if isinstance(v, set) else v for k, v in row.items()} for row in concept_summary] return concept_summary + def enrich_medcat_metrics(self, examples): + """ + Add the user prop to the medcat output metrics. Can potentially add more later for each of the categories + """ + for tp in [i for e_i in examples['tp'].values() for i in e_i]: + try: + ann = AnnotatedEntity.objects.get(project_id=tp['project id'], document_id=tp['document id'], + start_ind=tp['start'], end_ind=tp['end']) + tp['user'] = ann.user.username + except: + tp['user'] = None + for fp in (i for e_i in examples['fp'].values() for i in e_i): + try: + ann = AnnotatedEntity.objects.get(project_id=fp['project id'], document_id=fp['document id'], + start_ind=fp['start'], end_ind=fp['end']) + fp['user'] = ann.user.username + except: + fp['user'] = None + for fn in (i for e_i in examples['fn'].values() for i in e_i): + try: + ann = AnnotatedEntity.objects.get(project_id=fn['project id'], document_id=fn['document id'], + start_ind=fn['start'], end_ind=fn['end']) + fn['user'] = ann.user.username + except: + fn['user'] = None + return examples + def user_stats(self, by_user: bool = True): """ Summary of user annotation work done @@ -367,4 +400,8 @@ def generate_report(self, meta_ann=False): return {'user_stats': self.user_stats().to_dict('records'), 'concept_summary': self.concept_summary(), 'annotation_summary': anno_df.to_dict('records'), - 'meta_anno_summary': meta_anns_summary} + 'meta_anno_summary': meta_anns_summary, + 'projects2doc_ids': self.projects2doc_ids, + 'docs2text': self.docs2texts, + 'projects2name': self.projects2names, + 'docs2name': self.docs2names} diff --git a/webapp/frontend/package-lock.json b/webapp/frontend/package-lock.json index 79c4f254..d3d1df18 100644 --- a/webapp/frontend/package-lock.json +++ b/webapp/frontend/package-lock.json @@ -16,6 +16,7 @@ "@ssthouse/vue3-tree-chart": "^0.2.6", "axios": "^1.7.7", "bootstrap": "^5.3.3", + "plotly.js-dist": "^3.0.0", "splitpanes": "^3.1.5", "tiny-emitter": "^2.1.0", "v-runtime-template": "^1.10.0", @@ -1038,17 +1039,30 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.2.tgz", - "integrity": "sha512-CXtq5nR4Su+2I47WPOlWud98Y5Lv8Kyxp2ukhgFx/eW6Blm18VXJO5WuQylPugRo8nbluoi6GvvxBLqHcvqUUw==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.5.tgz", + "integrity": "sha512-lB05FkqEdUg2AA0xEbUz0SnkXT1LcCTa438W4IWTUh4hdOnVbQyOJ81OrDXsJk/LSiJHubgGEFoR5EHq1NsH1A==", "dev": true, "dependencies": { + "@eslint/core": "^0.10.0", "levn": "^0.4.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/plugin-kit/node_modules/@eslint/core": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.10.0.tgz", + "integrity": "sha512-gFHJ+xBOo4G3WRlR1e/3G8A6/KZAH6zcE/hkLRCZTi/B9avAG365QhFA8uOGzTMqgTghpn7/fSnscW++dpMSAw==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, "node_modules/@fortawesome/fontawesome-common-types": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.6.0.tgz", @@ -2711,9 +2725,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -4977,6 +4991,11 @@ "node": ">=0.10" } }, + "node_modules/plotly.js-dist": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/plotly.js-dist/-/plotly.js-dist-3.0.0.tgz", + "integrity": "sha512-CNr8bcGVOOjCqMD1t4KWy+7ikJOuxCK1nR0TRAmlKwz6tu+1oM0JlWgmRlRYlSgpqLJyEIvqNaO/Ajr268fmPw==" + }, "node_modules/postcss": { "version": "8.4.47", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz", diff --git a/webapp/frontend/package.json b/webapp/frontend/package.json index 3470d283..f3984df8 100644 --- a/webapp/frontend/package.json +++ b/webapp/frontend/package.json @@ -22,6 +22,7 @@ "@ssthouse/vue3-tree-chart": "^0.2.6", "axios": "^1.7.7", "bootstrap": "^5.3.3", + "plotly.js-dist": "^3.0.0", "splitpanes": "^3.1.5", "tiny-emitter": "^2.1.0", "v-runtime-template": "^1.10.0", diff --git a/webapp/frontend/src/components/anns/AnnoResult.vue b/webapp/frontend/src/components/anns/AnnoResult.vue index fd2b0658..a053e626 100644 --- a/webapp/frontend/src/components/anns/AnnoResult.vue +++ b/webapp/frontend/src/components/anns/AnnoResult.vue @@ -21,7 +21,8 @@ export default { type: { type: String, default: 'tp' - } + }, + docText: String }, computed: { text () { @@ -33,19 +34,11 @@ export default { if (this.type === 'fp' || this.type === 'fn') { highlightClass = 'highlight-task-1' } + const srcVal = this.result['source value'] - const resTxt = this.result.text - const regexp = RegExp(`${srcVal}`, 'sg') - const matches = [...resTxt.matchAll(regexp)] - let outText = '' - for (let match of matches) { - if (match.index === 60) { - // this is the match to use - other matches are spurious, and represent other MedCAT AnnoResults. - outText = `${resTxt.slice(0, match.index)}` - outText += `${srcVal}` - outText += `${resTxt.slice(match.index + srcVal.length)}` - } - } + let outText = `${this.docText.slice(this.result['start'] - 60, this.result['start'])}` + outText += `${srcVal}` + outText += `${this.docText.slice(this.result['end'], this.result['end'] + 60)}` return outText } }, diff --git a/webapp/frontend/src/views/Metrics.vue b/webapp/frontend/src/views/Metrics.vue index 25fdc9ea..9c668eab 100644 --- a/webapp/frontend/src/views/Metrics.vue +++ b/webapp/frontend/src/views/Metrics.vue @@ -34,7 +34,7 @@
- User Stats + Summary Stats Annotations Concept Summary Meta Annotations @@ -42,7 +42,38 @@
-
+
+ + + # Projects + {{ Object.keys(projects2name).length }} + + + + # Documents + {{ Object.values(projects2doc_ids).map(doc_ids => doc_ids.length).reduce((a, b) => a + b, 0) }} + + + + Annotation Overlap + {{ calculateOverlap(projects2doc_ids) }}% + + + + Annotator Agreement + {{ calculateAnnotatorAgreement(annoSummary) }} + + + + Total Annotation Time + {{ elapsedTime(annoSummary.items || []) }} + + + +
+
+
+
@@ -207,13 +238,13 @@
  • Alternative model predictions that are overlapping with other concepts
  • Genuine missed annotations by an annotator.
    • -

    Clicking through these annotations will not highlight this annotation as it doesn't exist in the +

    Clicking on these annotations will not highlight this annotation as it doesn't exist in the dataset

    False negative model predictions can be the result of:

      -
    • An model mistake that marked an annotation 'correct' where it should be incorrect
      • +
    • A model mistake that marked an annotation 'correct' where it should be incorrect
    • An annotator mistake that marked an annotation 'correct' where it should be incorrect
    @@ -229,7 +260,7 @@ + :type="modalData.type" :doc-text="textFromAnno(res)">
    @@ -241,6 +272,8 @@