Merge pull request #2445 from jjgao/merge-hotfix-to-master

1.6.1

business/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <artifactId>master</artifactId>
     <groupId>org.mskcc.cbio</groupId>
-    <version>1.6.0</version>
+    <version>1.6.1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>business</artifactId>

core/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <artifactId>master</artifactId>
     <groupId>org.mskcc.cbio</groupId>
-    <version>1.6.0</version>
+    <version>1.6.1</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>core</artifactId>

core/src/main/java/org/mskcc/cbio/portal/scripts/ConsoleRunnable.java

@@ -156,9 +156,8 @@ public void runInConsole() {
         }
         catch (Throwable t) {
             ConsoleUtil.showWarnings();
-            System.err.println ("\nABORTED! " + t.toString());
-            if (t.getMessage() == null)
-                t.printStackTrace();
+            System.err.println ("\nABORTED!");
+            t.printStackTrace();
             status = SysExit.select(t);
         }
         System.exit(status.getStatusCode());

core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanel.java

@@ -54,7 +54,6 @@ public void run() {
         try {
             String progName = "ImportGenePanel";
             String description = "Import gene panel files.";
-            // usage: --data <data_file.txt> --meta <meta_file.txt> --loadMode [directLoad|bulkLoad (default)] [--noprogress]
 
             OptionParser parser = new OptionParser();
             OptionSpec<String> data = parser.accepts( "data",
@@ -79,6 +78,7 @@ public void run() {
             }
 
             setFile(genePanel_f);
+            SpringUtil.initDataSource();
             importData();
         } catch (RuntimeException e) {
             throw e;

core/src/main/java/org/mskcc/cbio/portal/scripts/ImportGenePanelProfileMap.java

@@ -94,6 +94,7 @@ public void run() {
             }
 
             setFile(genePanel_f);
+            SpringUtil.initDataSource();
             importData();
         } catch (RuntimeException e) {
             throw e;

core/src/main/java/org/mskcc/cbio/portal/util/GlobalProperties.java

@@ -132,6 +132,7 @@ public class GlobalProperties {
     public static final String STUDY_VIEW_MDACC_HEATMAP_META_URL = "mdacc.heatmap.study.meta.url";
 
     public static final String ONCOKB_API_URL = "oncokb.api.url";
+    public static final String ONCOKB_PUBLIC_API_URL = "oncokb.public_api.url";
     public static final String SHOW_ONCOKB = "show.oncokb";
 
     private static String sessionServiceURL;
@@ -717,6 +718,29 @@ public static String getSessionServiceUrl()
     {
         return sessionServiceURL;
     }
+
+    public static String getOncoKBPublicApiUrl()
+    {
+        String oncokbApiUrl = properties.getProperty(ONCOKB_PUBLIC_API_URL);
+        String showOncokb = properties.getProperty(SHOW_ONCOKB);
+
+        if(showOncokb == null || showOncokb.isEmpty()) {
+                    showOncokb = "true";
+        }
+
+        // This only applies if there is no oncokb.api.url property in the portal.properties file.
+        // Empty string should be used if you want to disable the OncoKB annotation.
+        if(oncokbApiUrl == null || oncokbApiUrl.isEmpty()) {
+            oncokbApiUrl = "oncokb.org/api/v1";
+        }
+
+        if(showOncokb.equals("true")) {
+           return oncokbApiUrl;
+        } else {
+           return "";
+        }
+
+    }
 
     public static String getOncoKBApiUrl()
     {

core/src/main/scripts/importer/cbioportalImporter.py

@@ -108,6 +108,11 @@ def import_study_data(jvm_args, meta_filename, data_filename):
         args.append(data_filename)
         args.append("--study")
         args.append(meta_file_dict['cancer_study_identifier'])
+    elif importer == "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap":
+        args.append("--meta")
+        args.append(meta_filename)
+        args.append("--data")				
+        args.append(data_filename)
     else:
         args.append("--data")
         args.append(data_filename)
@@ -173,6 +178,7 @@ def process_directory(jvm_args, study_directory):
     cancer_type_filepairs = []
     sample_attr_filepair = None
     regular_filepairs = []
+    gene_panel_matrix_filepair = None
 
     # read all meta files (excluding case lists) to determine what to import
     for f in meta_filenames:
@@ -200,9 +206,12 @@ def process_directory(jvm_args, study_directory):
             if sample_attr_filepair is not None:
                 raise RuntimeError(
                     'Multiple sample attribute files found: {} and {}'.format(
-                        sample_attr_filepair[0], f))
+                        sample_attr_filepair[0], f))   # pylint: disable=unsubscriptable-object
             sample_attr_filepair = (
                 f, os.path.join(study_directory, metadata['data_filename']))
+        elif meta_file_type == MetaFileTypes.GENE_PANEL_MATRIX:
+            gene_panel_matrix_filepair = (
+                (f, os.path.join(study_directory, metadata['data_filename'])))
         else:
             regular_filepairs.append(
                 (f, os.path.join(study_directory, metadata['data_filename'])))
@@ -230,6 +239,9 @@ def process_directory(jvm_args, study_directory):
     for meta_filename, data_filename in regular_filepairs:
         import_study_data(jvm_args, meta_filename, data_filename)
 
+    if gene_panel_matrix_filepair is not None:
+        import_study_data(jvm_args, gene_panel_matrix_filepair[0], gene_panel_matrix_filepair[1])
+
     # do the case lists
     case_list_dirname = os.path.join(study_directory, 'case_lists')
     if os.path.isdir(case_list_dirname):

core/src/main/scripts/importer/cbioportal_common.py

@@ -46,6 +46,7 @@ class MetaFileTypes(object):
     TIMELINE = 'meta_timeline'
     CASE_LIST = 'case_list'
     MUTATION_SIGNIFICANCE = 'meta_mutsig'
+    GENE_PANEL_MATRIX = 'meta_gene_panel_matrix'
 
 # fields allowed in each meta file type, maps to True if required
 META_FIELD_MAP = {
@@ -85,7 +86,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.CNA_LOG2: {
         'cancer_study_identifier': True,
@@ -95,7 +97,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.CNA_CONTINUOUS: {
         'cancer_study_identifier': True,
@@ -105,7 +108,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.SEG: {
         'cancer_study_identifier': True,
@@ -125,7 +129,8 @@ class MetaFileTypes(object):
         'profile_description': True,
         'data_filename': True,
         'normal_samples_list': False,
-        'swissprot_identifier': False
+        'swissprot_identifier': False,
+        'gene_panel': False
     },
     MetaFileTypes.EXPRESSION: {
         'cancer_study_identifier': True,
@@ -135,7 +140,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.METHYLATION: {
         'cancer_study_identifier': True,
@@ -145,7 +151,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.PROTEIN: {
         'cancer_study_identifier': True,
@@ -155,7 +162,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.FUSION: {
         'cancer_study_identifier': True,
@@ -165,7 +173,8 @@ class MetaFileTypes(object):
         'show_profile_in_analysis_tab': True,
         'profile_name': True,
         'profile_description': True,
-        'data_filename': True
+        'data_filename': True,
+        'gene_panel': False
     },
     MetaFileTypes.GISTIC_GENES: {
         'cancer_study_identifier': True,
@@ -193,6 +202,12 @@ class MetaFileTypes(object):
         'genetic_alteration_type': True,
         'datatype': True,
         'data_filename': True
+    },
+    MetaFileTypes.GENE_PANEL_MATRIX: {
+        'cancer_study_identifier': True,
+        'genetic_alteration_type': True,
+        'datatype': True,
+        'data_filename': True
     }
 }
 
@@ -213,7 +228,8 @@ class MetaFileTypes(object):
     MetaFileTypes.GISTIC_GENES: "org.mskcc.cbio.portal.scripts.ImportGisticData",
     MetaFileTypes.TIMELINE: "org.mskcc.cbio.portal.scripts.ImportTimelineData",
     MetaFileTypes.CASE_LIST: IMPORT_CASE_LIST_CLASS,
-    MetaFileTypes.MUTATION_SIGNIFICANCE: "org.mskcc.cbio.portal.scripts.ImportMutSigData"
+    MetaFileTypes.MUTATION_SIGNIFICANCE: "org.mskcc.cbio.portal.scripts.ImportMutSigData",
+    MetaFileTypes.GENE_PANEL_MATRIX: "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap"
 }
 
 IMPORTER_REQUIRES_METADATA = {
@@ -222,7 +238,8 @@ class MetaFileTypes(object):
     "org.mskcc.cbio.portal.scripts.ImportGisticData" : False,
     "org.mskcc.cbio.portal.scripts.ImportMutSigData" : False,
     "org.mskcc.cbio.portal.scripts.ImportProfileData" : True,
-    "org.mskcc.cbio.portal.scripts.ImportTimelineData" : True
+    "org.mskcc.cbio.portal.scripts.ImportTimelineData" : True,
+    "org.mskcc.cbio.portal.scripts.ImportGenePanelProfileMap" : False
 }
 
 # ------------------------------------------------------------------------------
@@ -456,6 +473,7 @@ def get_meta_file_type(metaDictionary, logger, filename):
         # others
         ("METHYLATION", "CONTINUOUS"): MetaFileTypes.METHYLATION,
         ("FUSION", "FUSION"): MetaFileTypes.FUSION,
+        ("GENE_PANEL_MATRIX", "GENE_PANEL_MATRIX"): MetaFileTypes.GENE_PANEL_MATRIX,
         # cross-sample molecular statistics (for gene selection)
         ("GISTIC_GENES_AMP", "Q-VALUE"): MetaFileTypes.GISTIC_GENES,
         ("GISTIC_GENES_DEL", "Q-VALUE"): MetaFileTypes.GISTIC_GENES,

core/src/main/scripts/importer/validateData.py

@@ -71,7 +71,8 @@
     cbioportal_common.MetaFileTypes.PROTEIN:'ProteinLevelValidator',
     cbioportal_common.MetaFileTypes.GISTIC_GENES: 'GisticGenesValidator',
     cbioportal_common.MetaFileTypes.TIMELINE:'TimelineValidator',
-    cbioportal_common.MetaFileTypes.MUTATION_SIGNIFICANCE:'MutationSignificanceValidator'
+    cbioportal_common.MetaFileTypes.MUTATION_SIGNIFICANCE:'MutationSignificanceValidator',
+    cbioportal_common.MetaFileTypes.GENE_PANEL_MATRIX:'GenePanelMatrixValidator'
 }
 
 
@@ -175,7 +176,8 @@ def generateHtml(self):
         # refer to this function so that it can be used in the template:
         j_env.filters['os.path.relpath'] = os.path.relpath
         template = j_env.get_template('validation_report_template.html.jinja')
-        doc = template.render(
+        # pylint falsely infers template to be a string -- trust me, it's not
+        doc = template.render(   # pylint: disable=no-member
             study_dir=self.study_dir,
             cbio_version=self.cbio_version,
             record_list=self.buffer,
@@ -368,7 +370,7 @@ def _validate_file(self):
                     self.logger.info('Ignoring missing or invalid header comments. '
                         'Continuing with validation...')                    
                     self.fill_in_attr_defs = True
-
+            
             # read five data lines to detect quotes in the tsv file
             first_data_lines = []
             for i, line in enumerate(data_file):
@@ -455,7 +457,7 @@ def processTopLines(self, line_list):
         parsed, True otherwise.
         """
         return True
-
+    
     def checkHeader(self, cols):
 
         """Check that the header has the correct items and set self.cols.
@@ -2086,6 +2088,12 @@ class MutationSignificanceValidator(Validator):
     ALLOW_BLANKS = True
     pass
 
+class GenePanelMatrixValidator(Validator):
+
+    REQUIRED_HEADERS = ['SAMPLE_ID']
+    # TODO check that other column headers are valid profile stable ids
+    # TODO check that sample ids are references in clincal data file
+    # TODO check that referenced gene panel stable id is valid
 
 class ProteinLevelValidator(FeaturewiseFileValidator):
 

core/src/test/java/org/mskcc/cbio/portal/scripts/TestIntegrationTest.java

@@ -25,6 +25,7 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -54,6 +55,7 @@
 import org.mskcc.cbio.portal.model.*;
 import org.mskcc.cbio.portal.persistence.MutationMapperLegacy;
 import org.mskcc.cbio.portal.service.ApiService;
+import org.mskcc.cbio.portal.util.SpringUtil;
 import org.mskcc.cbio.portal.util.ConsoleUtil;
 import org.mskcc.cbio.portal.util.ProgressMonitor;
 import org.mskcc.cbio.portal.util.TransactionalScripts;
@@ -84,12 +86,14 @@ public class TestIntegrationTest {
     private ApplicationContext applicationContext;
 
     @Before
-    public void setUp() throws DaoException, JsonParseException, JsonMappingException, IOException {
+    public void setUp() throws DaoException, JsonParseException, JsonMappingException, IOException, Exception {
+        SpringUtil.setApplicationContext(applicationContext);
         ProgressMonitor.setConsoleMode(false);
         ProgressMonitor.resetWarnings();
         DaoCancerStudy.reCacheAll();
         DaoGeneOptimized.getInstance().reCache();
         loadGenes();
+        loadGenePanel();
     }
 
     /**
@@ -326,6 +330,16 @@ private void loadGenes() throws DaoException, JsonParseException, JsonMappingExc
         MySQLbulkLoader.flushAll();
 
     }
+
+    /**
+     * Loads a gene panel used by this test.
+     * 
+     */
+    private void loadGenePanel() throws Exception {
+        ImportGenePanel gp = new ImportGenePanel(null);
+        gp.setFile(new File("src/test/scripts/test_data/study_es_0/gene_panel_example.txt"));
+        gp.importData();
+    }
 
     @JsonIgnoreProperties(ignoreUnknown = true)
     static class TestGene {

core/src/test/scripts/test_data/study_es_0/data_gene_panel_matrix.txt

@@ -0,0 +1,2 @@
+SAMPLE_ID	study_es_0_mutations
+TCGA-A1-A0SB-01	example_gene_panel_stable_id

core/src/test/scripts/test_data/study_es_0/gene_panel_example.txt

@@ -0,0 +1,3 @@
+stable_id: example_gene_panel_stable_id
+description: Example gene panel meta file for testing purposes.
+gene_list:	AKT1	BRCA1	CDKN1A	EGFR	FOXP1	HRAS	KRAS	POLE	SMAD2	TP53

core/src/test/scripts/test_data/study_es_0/meta_fusion.txt

@@ -5,4 +5,5 @@ stable_id: fusion
 profile_description: Fusions.
 show_profile_in_analysis_tab: true
 profile_name: Fusions
-data_filename: data_fusions.txt
+data_filename: data_fusions.txt
+gene_panel: example_gene_panel_stable_id

core/src/test/scripts/test_data/study_es_0/meta_gene_panel_matrix.txt

@@ -0,0 +1,4 @@
+cancer_study_identifier: study_es_0
+genetic_alteration_type: GENE_PANEL_MATRIX
+datatype: GENE_PANEL_MATRIX
+data_filename: data_gene_panel_matrix.txt