-
Notifications
You must be signed in to change notification settings - Fork 597
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add reference and header parquet writers
- Loading branch information
1 parent
7a9d105
commit edd6938
Showing
9 changed files
with
482 additions
and
193 deletions.
There are no files selected for viewing
156 changes: 10 additions & 146 deletions
156
src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/CreateVariantIngestFiles.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 92 additions & 0 deletions
92
...main/java/org/broadinstitute/hellbender/utils/gvs/parquet/GvsHeaderParquetFileWriter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package org.broadinstitute.hellbender.utils.gvs.parquet; | ||
|
|
||
| import org.apache.hadoop.conf.Configuration; | ||
| import org.apache.hadoop.fs.FileAlreadyExistsException; | ||
| import org.apache.hadoop.fs.Path; | ||
| import org.apache.parquet.column.ParquetProperties; | ||
| import org.apache.parquet.hadoop.ParquetWriter; | ||
| import org.apache.parquet.hadoop.metadata.CompressionCodecName; | ||
| import org.apache.parquet.io.OutputFile; | ||
| import org.apache.parquet.schema.MessageType; | ||
| import org.json.JSONObject; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class GvsHeaderParquetFileWriter extends ParquetWriter<JSONObject> { | ||
|
|
||
| /** | ||
| * This is very deprecated, and we'll need to figure out how to do this from a builder once it works! | ||
| * @param file | ||
| * @param schema | ||
| * @param enableDictionary | ||
| * @param codecName | ||
| * @throws IOException | ||
| */ | ||
| public GvsHeaderParquetFileWriter( | ||
| Path file, | ||
| MessageType schema, | ||
| boolean enableDictionary, | ||
| CompressionCodecName codecName | ||
| ) throws FileAlreadyExistsException, IOException { | ||
| super(file, new GvsReferenceWriteSupport(schema), codecName, DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE, enableDictionary, false); | ||
| } | ||
|
|
||
| GvsHeaderParquetFileWriter( | ||
| Path file, | ||
| GvsVariantWriteSupport writeSupport, | ||
| CompressionCodecName compressionCodecName, | ||
| int blockSize, | ||
| int pageSize, | ||
| boolean enableDictionary, | ||
| boolean enableValidation, | ||
| ParquetProperties.WriterVersion writerVersion, | ||
| Configuration conf) | ||
| throws IOException { | ||
| super( | ||
| file, | ||
| writeSupport, | ||
| compressionCodecName, | ||
| blockSize, | ||
| pageSize, | ||
| pageSize, | ||
| enableDictionary, | ||
| enableValidation, | ||
| writerVersion, | ||
| conf); | ||
| } | ||
|
|
||
| public static JSONObject writeJson(Long sampleId, String headerLineHash) { | ||
| JSONObject record = new JSONObject(); | ||
| record.put("sample_id", sampleId); | ||
| record.put("headerLineHash", headerLineHash); | ||
| return record; | ||
| } | ||
|
|
||
| public static class Builder extends ParquetWriter.Builder<JSONObject, Builder> { | ||
| private MessageType schema = null; | ||
|
|
||
| private Builder(Path file) { | ||
| super(file); | ||
| } | ||
|
|
||
| private Builder(OutputFile file) { | ||
| super(file); | ||
| } | ||
|
|
||
| public Builder withType(MessageType type) { | ||
| this.schema = type; | ||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| protected Builder self() { | ||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| protected GvsVariantWriteSupport getWriteSupport(Configuration conf) { | ||
| return new GvsVariantWriteSupport(schema); | ||
| } | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.