Merge pull request #77 from wtsi-npg/devel

merge from devel for release 1.17

Changes

@@ -1,3 +1,9 @@
+release 1.17
+- New command line option ADD_CLUSTER_INDEX_TAG default false
+- Added function to read experiment name and computer name from runParameters file
+    used to generate read name prefix if config file does not exist, updated tests
+- Ignore N's in barcode sequence when looking for matches, updated tests
+
 release 1.16
 - AlignmentFilterTest.java, testChimericReads -
 -   compare deeply the content of json files with

nbproject/project.properties

@@ -33,7 +33,7 @@ file.reference.jackson-core-2.0.0.jar=lib/jackson/jackson-core-2.0.0.jar
 file.reference.jackson-databind-2.0.0.jar=lib/jackson/jackson-databind-2.0.0.jar
 file.reference.picard.jar=lib/picard/picard-1.96.jar
 file.reference.sam.jar=lib/picard/sam-1.96.jar
-illumina2bam.version=1.16
+illumina2bam.version=1.17
 includes=**
 jar.compress=false
 javac.classpath=\

src/uk/ac/sanger/npg/illumina/Illumina2bam.java

@@ -136,8 +136,9 @@ public class Illumina2bam extends PicardCommandLine {
             optional = true)
         public ArrayList<Integer> FINAL_INDEX_CYCLE;
 
-    //TODO: add command option to skip adding ci tag
-
+    @Option(shortName="CI", doc="Add cluster index tag, default false.", optional=true)
+    public boolean ADD_CLUSTER_INDEX_TAG = false;
+
 
     @Override
     protected int doWork() {
@@ -179,6 +180,7 @@ protected int doWork() {
                 this.LANE,
                 this.GENERATE_SECONDARY_BASE_CALLS,
                 this.PF_FILTER,
+                this.ADD_CLUSTER_INDEX_TAG,
                 this.OUTPUT,
                 this.BARCODE_SEQUENCE_TAG_NAME,
                 this.BARCODE_QUALITY_TAG_NAME,

src/uk/ac/sanger/npg/illumina/Lane.java

@@ -54,6 +54,7 @@ public class Lane {
     //fields must be given about what to output
     private final boolean includeSecondCall;
     private final boolean pfFilter;
+    private final boolean includeClusterIndex;
 
     //fields must be given for output bam
     private final File output;
@@ -109,6 +110,7 @@ public class Lane {
      * @param laneNumber lane number
      * @param secondCall including second base call or not, default false.
      * @param pfFilter Filter cluster or not, default true.
+     * @param clusterIndex add cluster index tag, default false.
      * @param output Output file
      * @param barcodeSeqTagName
      * @param barcodeQualTagName  
@@ -121,6 +123,7 @@ public Lane(String intensityDir,
                 int laneNumber,
                 boolean secondCall,
                 boolean pfFilter,
+                boolean clusterIndex,
                 File output,
                 String barcodeSeqTagName,
                 String barcodeQualTagName,
@@ -137,6 +140,7 @@ public Lane(String intensityDir,
         this.laneNumber        = laneNumber;
         this.includeSecondCall = secondCall;
         this.pfFilter          = pfFilter;
+        this.includeClusterIndex = clusterIndex;
         this.output            = output;
         this.barcodeSeqTagName  = barcodeSeqTagName;
         this.barcodeQualTagName = barcodeQualTagName;
@@ -186,6 +190,7 @@ public Lane(String intensityDir,
      * @param laneNumber lane number
      * @param secondCall including second base call or not, default false.
      * @param pfFilter Filter cluster or not, default true.
+     * @param clusterIndex add cluster index tag, default false.
      * @param output Output file
      * @param barcodeSeqTagName
      * @param barcodeQualTagName  
@@ -196,10 +201,11 @@ public Lane(String intensityDir,
                 int laneNumber,
                 boolean secondCall,
                 boolean pfFilter,
+                boolean clusterIndex,
                 File output,
                 String barcodeSeqTagName,
                 String barcodeQualTagName){
-        this(intensityDir, baseCallDir, runFolder, laneNumber, secondCall, pfFilter, output, barcodeSeqTagName, barcodeQualTagName, (String)null, (String)null);
+        this(intensityDir, baseCallDir, runFolder, laneNumber, secondCall, pfFilter, clusterIndex, output, barcodeSeqTagName, barcodeQualTagName, (String)null, (String)null);
     }
 
     /**
@@ -254,7 +260,7 @@ public boolean processTiles(SAMFileWriter outputSam) throws Exception{
 
             Tile tile = new Tile(intensityDir, baseCallDir, id, laneNumber, tileNumber,
                                  cycleRangeByRead,
-                                 this.includeSecondCall, this.pfFilter,
+                                 this.includeSecondCall, this.pfFilter, this.includeClusterIndex,
                                  this.barcodeSeqTagName, this.barcodeQualTagName);
 
             if(this.secondBarcodeSeqTagName != null && this.secondBarcodeQualTagName != null){
@@ -451,10 +457,20 @@ private void readRunConfig() throws Exception{
         this.id = this.readInstrumentAndRunID();
         if(id == null){
             log.warn("Problems to read run id and instrument name from config file:" + this.baseCallsConfig);
-            this.id = "";
         }else{
             log.info("Instrument name and run id to be used as part of read name: " + this.id );
         }
+        if(id == null){
+            this.id = this.readExperimentNameAndComputerName();
+            if(id == null){
+                log.warn("Problems to read experiment name and computer name from runParameter file");
+            }else{
+                log.info("Computer name and experiment name to be used as part of read name: " + this.id );
+            }
+        }
+        if(id == null){
+            this.id = "";
+        }
 
         runfolderConfig = readRunfolder();
         if(runfolderConfig == null && runParametersDoc != null){
@@ -712,6 +728,37 @@ public String readInstrumentAndRunID(){
         return instrument + "_" + runID;
     }
 
+    /**
+     *
+     * @return experiment name with computer name as part of read name, for example, HX10_15235
+     */
+    public String readExperimentNameAndComputerName(){
+
+        String experimentName = null;
+        String computerName   = null;
+        if(this.runParametersDoc != null){
+          try {
+              XPathExpression e = xpath.compile("RunParameters/Setup/ExperimentName/text()");
+              Node n = (Node) e.evaluate(this.runParametersDoc, XPathConstants.NODE);
+              if (n != null) {
+                  experimentName = n.getNodeValue();
+                  e = xpath.compile("RunParameters/Setup/ComputerName/text()");
+                  n = (Node) e.evaluate(this.runParametersDoc, XPathConstants.NODE);
+                  if (n != null) {
+                      computerName = n.getNodeValue();
+                  }
+              }
+          } catch (XPathExpressionException ex) {
+              log.error("Problems to read experiment name and computer name from run parameter file: " + ex.getMessage() );
+          }
+        }
+        if(experimentName == null || computerName == null) {
+            log.warn("No experiment name or computer name returned.");
+            return null;
+        }
+        return computerName + "_" + experimentName;
+    }
+
     /**
      * 
      * @return

src/uk/ac/sanger/npg/illumina/Tile.java

@@ -58,6 +58,7 @@ public class Tile {
 
     private final boolean includeSecondCall;
     private final boolean pfFilter;
+    private final boolean includeClusterIndex;
 
     //temp fields    
     private final String laneSubDir;
@@ -87,6 +88,7 @@ public class Tile {
      * @param cycleRangeByRead cycle range for each read, the hash key could be read1, read2 or readIndex
      * @param secondCall include second base call or not
      * @param pfFilter include PF filtered reads or not
+     * @param clusterIndex add cluter index tag, default false.
      * @param barcodeSeqTagName
      * @param barcodeQualTagName  
      */
@@ -98,6 +100,7 @@ public Tile(String intensityDir,
             HashMap<String, int[]> cycleRangeByRead,
             boolean secondCall,
             boolean pfFilter,
+            boolean clusterIndex,
             String barcodeSeqTagName,
             String barcodeQualTagName) {
 
@@ -112,6 +115,7 @@ public Tile(String intensityDir,
 
         this.includeSecondCall = secondCall;
         this.pfFilter = pfFilter;
+        this.includeClusterIndex = clusterIndex;
 
         this.cycleRangeByRead = cycleRangeByRead;
 
@@ -569,7 +573,11 @@ public SAMRecord getSAMRecord(
         SAMRecord samRecord = new SAMRecord(fileHeader);
 
         samRecord.setReadName(readName);
-        samRecord.setAttribute("ci", clusterIndex);
+
+        if(this.includeClusterIndex) {
+            samRecord.setAttribute("ci", clusterIndex);
+        }
+
         samRecord.setReadBases(baseQuals[0]);
         samRecord.setBaseQualities(baseQuals[1]);
         samRecord.setReadUnmappedFlag(true);
@@ -769,7 +777,7 @@ public static void main (String [] args) throws Exception{
         cycleRangeByRead.put("read2", cycleRangeRead2);
         cycleRangeByRead.put("readIndex", cycleRangeIndex);
 
-        Tile tile = new Tile(intensityDir, baseCallDir, id, lane, tileNumber, cycleRangeByRead, true, true, barcodeSeqTagName, barcodeQualTagName);
+        Tile tile = new Tile(intensityDir, baseCallDir, id, lane, tileNumber, cycleRangeByRead, true, true, true, barcodeSeqTagName, barcodeQualTagName);
 
         File outBam = new File("test.bam");
         SAMFileWriterFactory factory = new SAMFileWriterFactory();

src/uk/ac/sanger/npg/picard/IndexDecoder.java

@@ -302,7 +302,7 @@ private BarcodeMatch findBestBarcode(final String readSubsequence, final boolean
     private int countMismatches(final byte[] barcodeBytes, final byte[] readSubsequence) {
         int numMismatches = 0;
         for (int i = 0; i < barcodeBytes.length; ++i) {
-            if (!SequenceUtil.isNoCall(readSubsequence[i]) && !SequenceUtil.basesEqual(barcodeBytes[i], readSubsequence[i])) {
+            if (!SequenceUtil.isNoCall(readSubsequence[i]) && !SequenceUtil.isNoCall(barcodeBytes[i]) && !SequenceUtil.basesEqual(barcodeBytes[i], readSubsequence[i])) {
                 ++numMismatches;
             }
         }

src/uk/ac/sanger/npg/picard/PicardCommandLine.java

@@ -32,7 +32,7 @@
  */
 public abstract class PicardCommandLine extends CommandLineProgram {
 
-    public static final String version = "1.16";
+    public static final String version = "1.17";
 
     /**
      * Generate Program Record for this program itself

test/uk/ac/sanger/npg/illumina/Illumina2bamTest.java

@@ -96,7 +96,7 @@ public void testMain() {
                 + " TMP_DIR=[testdata] VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=1"
                 + " CREATE_MD5_FILE=true    GENERATE_SECONDARY_BASE_CALLS=false PF_FILTER=true READ_GROUP_ID=1"
                 + " SEQUENCING_CENTER=SC PLATFORM=ILLUMINA BARCODE_SEQUENCE_TAG_NAME=BC BARCODE_QUALITY_TAG_NAME=QT"
-                + " VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
+                + " ADD_CLUSTER_INDEX_TAG=false VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
                 testData.illumina2bam.getCommandLine()
                );
 
@@ -108,7 +108,7 @@ public void testMain() {
         assertEquals("Ensure PG field of RG record corresponds to the appropriate program", pgr.getId(),  rgl.get(0).getAttribute("PG"));
         samFileReader.close();
 
-        assertEquals("ce104cf5c591cdbf94eb7b584d6f5352", CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("e614c4c92250472f1f01ba87bb7673e5", CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 
     /**
@@ -144,11 +144,11 @@ public void noFirstTileTest() {
                 + " TMP_DIR=[testdata] VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=1"
                 + " CREATE_MD5_FILE=true    GENERATE_SECONDARY_BASE_CALLS=false PF_FILTER=true READ_GROUP_ID=1"
                 + " SEQUENCING_CENTER=SC PLATFORM=ILLUMINA BARCODE_SEQUENCE_TAG_NAME=BC BARCODE_QUALITY_TAG_NAME=QT"
-                + " VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
+                + " ADD_CLUSTER_INDEX_TAG=false VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
                 testData.illumina2bam.getCommandLine()
                );
 
-        assertEquals("1ce96b8c94224cfaa260e871f2e997dc", CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("d86e325da70ac9fe580baec4f393d518", CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 
     /**
@@ -186,11 +186,11 @@ public void cycleRangeTest() {
                 + " TMP_DIR=[testdata] VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=1"
                 + " CREATE_MD5_FILE=true    GENERATE_SECONDARY_BASE_CALLS=false PF_FILTER=true READ_GROUP_ID=1"
                 + " SEQUENCING_CENTER=SC PLATFORM=ILLUMINA BARCODE_SEQUENCE_TAG_NAME=BC BARCODE_QUALITY_TAG_NAME=QT"
-                + " VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
+                + " ADD_CLUSTER_INDEX_TAG=false VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
                 testData.illumina2bam.getCommandLine()
                );
 
-        assertEquals("f8452061bbc72a2dbfb4eda3d5ed896a",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("9671152451a948627c3de6a57c9263fc",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 
     /**
@@ -233,11 +233,11 @@ public void bcReadTest() {
                 + " TMP_DIR=[testdata] VALIDATION_STRINGENCY=STRICT COMPRESSION_LEVEL=1"
                 + " CREATE_MD5_FILE=true    GENERATE_SECONDARY_BASE_CALLS=false PF_FILTER=true READ_GROUP_ID=1"
                 + " SEQUENCING_CENTER=SC PLATFORM=ILLUMINA BARCODE_SEQUENCE_TAG_NAME=BC BARCODE_QUALITY_TAG_NAME=QT"
-                + " VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
+                + " ADD_CLUSTER_INDEX_TAG=false VERBOSITY=INFO QUIET=false MAX_RECORDS_IN_RAM=500000 CREATE_INDEX=false",
                 testData.illumina2bam.getCommandLine()
                );
 
-        assertEquals("c31d1cc84eff1ab6ce5ba5fa462fe014",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("a537f56354ab6051f0766c3d1ad77993",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 
     /**
@@ -263,7 +263,7 @@ public void dualIndexRunTest() {
                 "PF_FILTER=false"
                };
         testData.commonAsserts(args);        
-        assertEquals("2c0d8b137bd8fe5afe5e50c11b561b0f",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("8135dfaa03f248bb23064cd121a4a298",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 
     /**
@@ -290,6 +290,6 @@ public void propagationOfSecondIndexParameters() {
                 "BC_SEQ=tr","BC_QUAL=tq","SEC_BC_SEQ=BC","SEC_BC_QUAL=QT"
                };
         testData.commonAsserts(args);        
-        assertEquals("cc191980b6d85386ae003971b02a3cc9",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
+        assertEquals("534d7d7183ccdeead103a669b7105cb2",CheckMd5.getBamMd5AfterRemovePGVersion(testData.tempBamFile, "Illumina2bam"));
     }
 }