diff --git a/business/pom.xml b/business/pom.xml index efe733900f1..7aa2f50d793 100644 --- a/business/pom.xml +++ b/business/pom.xml @@ -4,7 +4,7 @@ master org.mskcc.cbio - 1.6.0 + 1.6.1 4.0.0 business diff --git a/core/pom.xml b/core/pom.xml index a690df1db82..ca6372104fc 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ master org.mskcc.cbio - 1.6.0 + 1.6.1 4.0.0 core diff --git a/core/src/main/java/org/mskcc/cbio/portal/scripts/ConsoleRunnable.java b/core/src/main/java/org/mskcc/cbio/portal/scripts/ConsoleRunnable.java index 59e19a91e9f..0ce5e15e3d4 100644 --- a/core/src/main/java/org/mskcc/cbio/portal/scripts/ConsoleRunnable.java +++ b/core/src/main/java/org/mskcc/cbio/portal/scripts/ConsoleRunnable.java @@ -156,9 +156,8 @@ public void runInConsole() { } catch (Throwable t) { ConsoleUtil.showWarnings(); - System.err.println ("\nABORTED! " + t.toString()); - if (t.getMessage() == null) - t.printStackTrace(); + System.err.println ("\nABORTED!"); + t.printStackTrace(); status = SysExit.select(t); } System.exit(status.getStatusCode()); diff --git a/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanel.java b/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanel.java index b62eab2f52e..17da88df0a4 100644 --- a/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanel.java +++ b/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanel.java @@ -54,7 +54,6 @@ public void run() { try { String progName = "ImportGenePanel"; String description = "Import gene panel files."; - // usage: --data --meta --loadMode [directLoad|bulkLoad (default)] [--noprogress] OptionParser parser = new OptionParser(); OptionSpec data = parser.accepts( "data", @@ -79,6 +78,7 @@ public void run() { } setFile(genePanel_f); + SpringUtil.initDataSource(); importData(); } catch (RuntimeException e) { throw e; diff --git a/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java b/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java index a3e227c3d9e..adb5ad2358d 100644 --- a/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java +++ b/core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java @@ -94,6 +94,7 @@ public void run() { } setFile(genePanel_f); + SpringUtil.initDataSource(); importData(); } catch (RuntimeException e) { throw e; diff --git a/core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java b/core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java index 664748ab029..36f2fe01105 100644 --- a/core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java +++ b/core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java @@ -132,6 +132,7 @@ public class GlobalProperties { public static final String STUDY_VIEW_MDACC_HEATMAP_META_URL = "mdacc.heatmap.study.meta.url"; public static final String ONCOKB_API_URL = "oncokb.api.url"; + public static final String ONCOKB_PUBLIC_API_URL = "oncokb.public_api.url"; public static final String SHOW_ONCOKB = "show.oncokb"; private static String sessionServiceURL; @@ -717,6 +718,29 @@ public static String getSessionServiceUrl() { return sessionServiceURL; } + + public static String getOncoKBPublicApiUrl() + { + String oncokbApiUrl = properties.getProperty(ONCOKB_PUBLIC_API_URL); + String showOncokb = properties.getProperty(SHOW_ONCOKB); + + if(showOncokb == null || showOncokb.isEmpty()) { + showOncokb = "true"; + } + + // This only applies if there is no oncokb.api.url property in the portal.properties file. + // Empty string should be used if you want to disable the OncoKB annotation. + if(oncokbApiUrl == null || oncokbApiUrl.isEmpty()) { + oncokbApiUrl = "oncokb.org/api/v1"; + } + + if(showOncokb.equals("true")) { + return oncokbApiUrl; + } else { + return ""; + } + + } public static String getOncoKBApiUrl() { diff --git a/core/src/main/scripts/importer/cbioportalImporter.py b/core/src/main/scripts/importer/cbioportalImporter.py index 3db0010bf14..9fa04a48ee4 100755 --- a/core/src/main/scripts/importer/cbioportalImporter.py +++ b/core/src/main/scripts/importer/cbioportalImporter.py @@ -108,6 +108,11 @@ def import_study_data(jvm_args, meta_filename, data_filename): args.append(data_filename) args.append("--study") args.append(meta_file_dict['cancer_study_identifier']) + elif importer == "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap": + args.append("--meta") + args.append(meta_filename) + args.append("--data") + args.append(data_filename) else: args.append("--data") args.append(data_filename) @@ -173,6 +178,7 @@ def process_directory(jvm_args, study_directory): cancer_type_filepairs = [] sample_attr_filepair = None regular_filepairs = [] + gene_panel_matrix_filepair = None # read all meta files (excluding case lists) to determine what to import for f in meta_filenames: @@ -200,9 +206,12 @@ def process_directory(jvm_args, study_directory): if sample_attr_filepair is not None: raise RuntimeError( 'Multiple sample attribute files found: {} and {}'.format( - sample_attr_filepair[0], f)) + sample_attr_filepair[0], f)) # pylint: disable=unsubscriptable-object sample_attr_filepair = ( f, os.path.join(study_directory, metadata['data_filename'])) + elif meta_file_type == MetaFileTypes.GENE_PANEL_MATRIX: + gene_panel_matrix_filepair = ( + (f, os.path.join(study_directory, metadata['data_filename']))) else: regular_filepairs.append( (f, os.path.join(study_directory, metadata['data_filename']))) @@ -230,6 +239,9 @@ def process_directory(jvm_args, study_directory): for meta_filename, data_filename in regular_filepairs: import_study_data(jvm_args, meta_filename, data_filename) + if gene_panel_matrix_filepair is not None: + import_study_data(jvm_args, gene_panel_matrix_filepair[0], gene_panel_matrix_filepair[1]) + # do the case lists case_list_dirname = os.path.join(study_directory, 'case_lists') if os.path.isdir(case_list_dirname): diff --git a/core/src/main/scripts/importer/cbioportal_common.py b/core/src/main/scripts/importer/cbioportal_common.py index 9176b4123e0..affb278cefd 100644 --- a/core/src/main/scripts/importer/cbioportal_common.py +++ b/core/src/main/scripts/importer/cbioportal_common.py @@ -46,6 +46,7 @@ class MetaFileTypes(object): TIMELINE = 'meta_timeline' CASE_LIST = 'case_list' MUTATION_SIGNIFICANCE = 'meta_mutsig' + GENE_PANEL_MATRIX = 'meta_gene_panel_matrix' # fields allowed in each meta file type, maps to True if required META_FIELD_MAP = { @@ -85,7 +86,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.CNA_LOG2: { 'cancer_study_identifier': True, @@ -95,7 +97,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.CNA_CONTINUOUS: { 'cancer_study_identifier': True, @@ -105,7 +108,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.SEG: { 'cancer_study_identifier': True, @@ -125,7 +129,8 @@ class MetaFileTypes(object): 'profile_description': True, 'data_filename': True, 'normal_samples_list': False, - 'swissprot_identifier': False + 'swissprot_identifier': False, + 'gene_panel': False }, MetaFileTypes.EXPRESSION: { 'cancer_study_identifier': True, @@ -135,7 +140,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.METHYLATION: { 'cancer_study_identifier': True, @@ -145,7 +151,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.PROTEIN: { 'cancer_study_identifier': True, @@ -155,7 +162,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.FUSION: { 'cancer_study_identifier': True, @@ -165,7 +173,8 @@ class MetaFileTypes(object): 'show_profile_in_analysis_tab': True, 'profile_name': True, 'profile_description': True, - 'data_filename': True + 'data_filename': True, + 'gene_panel': False }, MetaFileTypes.GISTIC_GENES: { 'cancer_study_identifier': True, @@ -193,6 +202,12 @@ class MetaFileTypes(object): 'genetic_alteration_type': True, 'datatype': True, 'data_filename': True + }, + MetaFileTypes.GENE_PANEL_MATRIX: { + 'cancer_study_identifier': True, + 'genetic_alteration_type': True, + 'datatype': True, + 'data_filename': True } } @@ -213,7 +228,8 @@ class MetaFileTypes(object): MetaFileTypes.GISTIC_GENES: "org.mskcc.cbio.portal.scripts.ImportGisticData", MetaFileTypes.TIMELINE: "org.mskcc.cbio.portal.scripts.ImportTimelineData", MetaFileTypes.CASE_LIST: IMPORT_CASE_LIST_CLASS, - MetaFileTypes.MUTATION_SIGNIFICANCE: "org.mskcc.cbio.portal.scripts.ImportMutSigData" + MetaFileTypes.MUTATION_SIGNIFICANCE: "org.mskcc.cbio.portal.scripts.ImportMutSigData", + MetaFileTypes.GENE_PANEL_MATRIX: "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap" } IMPORTER_REQUIRES_METADATA = { @@ -222,7 +238,8 @@ class MetaFileTypes(object): "org.mskcc.cbio.portal.scripts.ImportGisticData" : False, "org.mskcc.cbio.portal.scripts.ImportMutSigData" : False, "org.mskcc.cbio.portal.scripts.ImportProfileData" : True, - "org.mskcc.cbio.portal.scripts.ImportTimelineData" : True + "org.mskcc.cbio.portal.scripts.ImportTimelineData" : True, + "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap" : False } # ------------------------------------------------------------------------------ @@ -456,6 +473,7 @@ def get_meta_file_type(metaDictionary, logger, filename): # others ("METHYLATION", "CONTINUOUS"): MetaFileTypes.METHYLATION, ("FUSION", "FUSION"): MetaFileTypes.FUSION, + ("GENE_PANEL_MATRIX", "GENE_PANEL_MATRIX"): MetaFileTypes.GENE_PANEL_MATRIX, # cross-sample molecular statistics (for gene selection) ("GISTIC_GENES_AMP", "Q-VALUE"): MetaFileTypes.GISTIC_GENES, ("GISTIC_GENES_DEL", "Q-VALUE"): MetaFileTypes.GISTIC_GENES, diff --git a/core/src/main/scripts/importer/validateData.py b/core/src/main/scripts/importer/validateData.py index ea833647c38..ec348df3036 100755 --- a/core/src/main/scripts/importer/validateData.py +++ b/core/src/main/scripts/importer/validateData.py @@ -71,7 +71,8 @@ cbioportal_common.MetaFileTypes.PROTEIN:'ProteinLevelValidator', cbioportal_common.MetaFileTypes.GISTIC_GENES: 'GisticGenesValidator', cbioportal_common.MetaFileTypes.TIMELINE:'TimelineValidator', - cbioportal_common.MetaFileTypes.MUTATION_SIGNIFICANCE:'MutationSignificanceValidator' + cbioportal_common.MetaFileTypes.MUTATION_SIGNIFICANCE:'MutationSignificanceValidator', + cbioportal_common.MetaFileTypes.GENE_PANEL_MATRIX:'GenePanelMatrixValidator' } @@ -175,7 +176,8 @@ def generateHtml(self): # refer to this function so that it can be used in the template: j_env.filters['os.path.relpath'] = os.path.relpath template = j_env.get_template('validation_report_template.html.jinja') - doc = template.render( + # pylint falsely infers template to be a string -- trust me, it's not + doc = template.render( # pylint: disable=no-member study_dir=self.study_dir, cbio_version=self.cbio_version, record_list=self.buffer, @@ -368,7 +370,7 @@ def _validate_file(self): self.logger.info('Ignoring missing or invalid header comments. ' 'Continuing with validation...') self.fill_in_attr_defs = True - + # read five data lines to detect quotes in the tsv file first_data_lines = [] for i, line in enumerate(data_file): @@ -455,7 +457,7 @@ def processTopLines(self, line_list): parsed, True otherwise. """ return True - + def checkHeader(self, cols): """Check that the header has the correct items and set self.cols. @@ -2086,6 +2088,12 @@ class MutationSignificanceValidator(Validator): ALLOW_BLANKS = True pass +class GenePanelMatrixValidator(Validator): + + REQUIRED_HEADERS = ['SAMPLE_ID'] + # TODO check that other column headers are valid profile stable ids + # TODO check that sample ids are references in clincal data file + # TODO check that referenced gene panel stable id is valid class ProteinLevelValidator(FeaturewiseFileValidator): diff --git a/core/src/test/java/org/mskcc/cbio/portal/scripts/TestIntegrationTest.java b/core/src/test/java/org/mskcc/cbio/portal/scripts/TestIntegrationTest.java index c286743a745..626f3bcf05c 100644 --- a/core/src/test/java/org/mskcc/cbio/portal/scripts/TestIntegrationTest.java +++ b/core/src/test/java/org/mskcc/cbio/portal/scripts/TestIntegrationTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -54,6 +55,7 @@ import org.mskcc.cbio.portal.model.*; import org.mskcc.cbio.portal.persistence.MutationMapperLegacy; import org.mskcc.cbio.portal.service.ApiService; +import org.mskcc.cbio.portal.util.SpringUtil; import org.mskcc.cbio.portal.util.ConsoleUtil; import org.mskcc.cbio.portal.util.ProgressMonitor; import org.mskcc.cbio.portal.util.TransactionalScripts; @@ -84,12 +86,14 @@ public class TestIntegrationTest { private ApplicationContext applicationContext; @Before - public void setUp() throws DaoException, JsonParseException, JsonMappingException, IOException { + public void setUp() throws DaoException, JsonParseException, JsonMappingException, IOException, Exception { + SpringUtil.setApplicationContext(applicationContext); ProgressMonitor.setConsoleMode(false); ProgressMonitor.resetWarnings(); DaoCancerStudy.reCacheAll(); DaoGeneOptimized.getInstance().reCache(); loadGenes(); + loadGenePanel(); } /** @@ -326,6 +330,16 @@ private void loadGenes() throws DaoException, JsonParseException, JsonMappingExc MySQLbulkLoader.flushAll(); } + + /** + * Loads a gene panel used by this test. + * + */ + private void loadGenePanel() throws Exception { + ImportGenePanel gp = new ImportGenePanel(null); + gp.setFile(new File("src/test/scripts/test_data/study_es_0/gene_panel_example.txt")); + gp.importData(); + } @JsonIgnoreProperties(ignoreUnknown = true) static class TestGene { diff --git a/core/src/test/scripts/test_data/study_es_0/data_gene_panel_matrix.txt b/core/src/test/scripts/test_data/study_es_0/data_gene_panel_matrix.txt new file mode 100644 index 00000000000..56301b52ffc --- /dev/null +++ b/core/src/test/scripts/test_data/study_es_0/data_gene_panel_matrix.txt @@ -0,0 +1,2 @@ +SAMPLE_ID study_es_0_mutations +TCGA-A1-A0SB-01 example_gene_panel_stable_id diff --git a/core/src/test/scripts/test_data/study_es_0/gene_panel_example.txt b/core/src/test/scripts/test_data/study_es_0/gene_panel_example.txt new file mode 100644 index 00000000000..44fc582b6da --- /dev/null +++ b/core/src/test/scripts/test_data/study_es_0/gene_panel_example.txt @@ -0,0 +1,3 @@ +stable_id: example_gene_panel_stable_id +description: Example gene panel meta file for testing purposes. +gene_list: AKT1 BRCA1 CDKN1A EGFR FOXP1 HRAS KRAS POLE SMAD2 TP53 diff --git a/core/src/test/scripts/test_data/study_es_0/meta_fusion.txt b/core/src/test/scripts/test_data/study_es_0/meta_fusion.txt index 1696a4bfc81..195fff714c3 100644 --- a/core/src/test/scripts/test_data/study_es_0/meta_fusion.txt +++ b/core/src/test/scripts/test_data/study_es_0/meta_fusion.txt @@ -5,4 +5,5 @@ stable_id: fusion profile_description: Fusions. show_profile_in_analysis_tab: true profile_name: Fusions -data_filename: data_fusions.txt \ No newline at end of file +data_filename: data_fusions.txt +gene_panel: example_gene_panel_stable_id diff --git a/core/src/test/scripts/test_data/study_es_0/meta_gene_panel_matrix.txt b/core/src/test/scripts/test_data/study_es_0/meta_gene_panel_matrix.txt new file mode 100644 index 00000000000..440f19c3d84 --- /dev/null +++ b/core/src/test/scripts/test_data/study_es_0/meta_gene_panel_matrix.txt @@ -0,0 +1,4 @@ +cancer_study_identifier: study_es_0 +genetic_alteration_type: GENE_PANEL_MATRIX +datatype: GENE_PANEL_MATRIX +data_filename: data_gene_panel_matrix.txt diff --git a/core/src/test/scripts/test_data/study_es_0/result_report.html b/core/src/test/scripts/test_data/study_es_0/result_report.html index 2e931d5235d..eb417c7bead 100644 --- a/core/src/test/scripts/test_data/study_es_0/result_report.html +++ b/core/src/test/scripts/test_data/study_es_0/result_report.html @@ -575,6 +575,57 @@

data_fusions.txt

+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Line NumberColumn NumberMessageValue Encountered
Debug––Starting validation of file–
Info––Validation of file complete–
Info––Read 2 lines. Lines with warning: 0. Lines with error: 0–
+
+
+
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Line NumberColumn NumberMessageValue Encountered
Debug––Starting validation of meta file–
Info––Validation of meta file complete–
+
+
+
+