From 3f9df6d5ae18ef12f163a6e6fae0e56d27d24b9a Mon Sep 17 00:00:00 2001 From: Sven F Date: Wed, 27 Dec 2017 09:28:46 +0100 Subject: [PATCH 01/15] Update README #24 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d2cca56..b8a54ae 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ [![Build Status](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/qPostMan-development/badge/icon)](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/qPostMan-development/) # qPostMan + +**Current stable version: 0.1.2.3** + A client software written in Java for dataset downloads from QBiC's data management system openBIS (https://wiki-bsse.ethz.ch/display/bis/Home). We are making use of the V3 API of openBIS (https://wiki-bsse.ethz.ch/display/openBISDoc1605/openBIS+V3+API) in order to interact with the data management system from command line, in order to provide a quick data retreaval on server or cluster resources, where the download via the qPortal is impractical. @@ -42,7 +45,7 @@ qPostMan will prompt you for your password, which is the password from your QBiC After you have provided your password and authenticate successfully, qPostMan tries to download all datasets that are registered for that given sample ID and downloads them to the current working directory: ```bash -[bbbfs01@u-003-ncmu03 ~]$ qpostman-0.1.2.1 -i QMFKD003AG -u bbbfs01 +[bbbfs01@u-003-ncmu03 ~]$ qpostman-0.1.2.3 -i QMFKD003AG -u bbbfs01 Provide password for user 'bbbfs01': 12:32:02.038 [main] INFO life.qbic.App - OpenBis login returned with 0 From d24f5c32516da002e6367308f6239c9f1009c21d Mon Sep 17 00:00:00 2001 From: sven1103 Date: Mon, 23 Apr 2018 16:31:47 +0200 Subject: [PATCH 02/15] Change CLI library to picocli --- pom.xml | 129 +++++++++++--------- src/main/java/life/qbic/App.java | 40 ++---- src/main/java/life/qbic/QbicDataLoader.java | 4 +- 3 files changed, 85 insertions(+), 88 deletions(-) diff --git a/pom.xml b/pom.xml index 2a8ff80..659006d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,45 +4,56 @@ life.qbic qpostman jar - 0.1.2.3 + 0.1.3 qpostman http://maven.apache.org 1.8 1.8 + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.0 + + + junit + junit + 3.8.1 + test + + + life.qbic.openbis + openbis_api + 3-S253.0 + + + + commons-cli + commons-cli + 1.4 + + + + org.apache.logging.log4j + log4j-core + 2.9.1 + + + jline + jline + 2.14.2 + + + + info.picocli + picocli + 2.3.0 + + - - - junit - junit - 3.8.1 - test - - - life.qbic.openbis - openbis_api - 3-S253.0 - - - - commons-cli - commons-cli - 1.4 - - - - org.apache.logging.log4j - log4j-core - 2.9.1 - - - jline - jline - 2.14.2 - - - @@ -61,31 +72,31 @@ - - - nexus-snap - qbic snapshots - https://qbic-repo.am10.uni-tuebingen.de/repository/maven-snapshots - - - nexus-release - qbic release - https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases - - - - - true - nexus-release - QBiC Releases - https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases - - - false - nexus-snap - QBiC Snapshots - https://qbic-repo.am10.uni-tuebingen.de/repository/maven-snapshots - + + + nexus-snap + qbic snapshots + https://qbic-repo.am10.uni-tuebingen.de/repository/maven-snapshots + + + nexus-release + qbic release + https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases + + + + + true + nexus-release + QBiC Releases + https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases + + + false + nexus-snap + QBiC Snapshots + https://qbic-repo.am10.uni-tuebingen.de/repository/maven-snapshots + - - + + diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 0b50fb0..ac1ef27 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import picocli.CommandLine; @@ -23,45 +24,30 @@ public class App { static String DSS_URL = "https://qbis.qbic.uni-tuebingen.de:444/datastore_server"; static Logger log = LogManager.getLogger(App.class); - public static void main(String[] args) throws IOException { - - Map cmdValues = Argparser.parseCmdArguments(args); - - String user = cmdValues.get(Argparser.Attribute.USERNAME); - - String id = cmdValues.get(Argparser.Attribute.ID); - - String filePath = cmdValues.get(Argparser.Attribute.FILE); - List identifiers = new ArrayList(); + public static void main(String[] args) throws IOException { - if (cmdValues.containsKey(Argparser.Attribute.HELP)) { - Argparser.printHelp(); - System.exit(0); - } + MyCommandLine commandLine = new MyCommandLine(); + new CommandLine(commandLine).parse(args); - if (user == null) { + if (commandLine.user == null) { Argparser.printHelp(); System.exit(1); } - if ((id == null || id.isEmpty()) && (filePath == null || filePath.isEmpty())) { + if ((commandLine.ids == null || commandLine.ids.isEmpty()) && (commandLine.filePath == null || commandLine.filePath == null)) { System.out .println("You have to provide one ID as command line argument or a file containing IDs."); - Argparser.printHelp(); System.exit(1); - } else if ((id != null) && (filePath != null)) { + } else if ((commandLine.ids != null) && (commandLine.filePath != null)) { System.out.println( "Arguments --identifier and --file are mutually exclusive, please provide only one."); - Argparser.printHelp(); System.exit(1); - } else if ((id != null)) { - identifiers.add(id); - } else { - identifiers.addAll(Argparser.readProvidedIndentifiers(new File(filePath))); + } else if (commandLine.filePath != null) { + commandLine.ids = Argparser.readProvidedIndentifiers(commandLine.filePath.toFile()); } - System.out.format("Provide password for user \'%s\':\n", user); + System.out.format("Provide password for user \'%s\':\n", commandLine.user); String password = Argparser.readPasswordFromInputStream(); @@ -70,7 +56,7 @@ public static void main(String[] args) throws IOException { System.exit(1); } - QbicDataLoader qbicDataLoader = new QbicDataLoader(AS_URL, DSS_URL, user, password, cmdValues.get(Argparser.Attribute.BUFFER_SIZE)); + QbicDataLoader qbicDataLoader = new QbicDataLoader(AS_URL, DSS_URL, commandLine.user, password, commandLine.bufferMultiplier*1024); int returnCode = qbicDataLoader.login(); log.info(String.format("OpenBis login returned with %s", returnCode)); if (returnCode != 0) { @@ -80,9 +66,9 @@ public static void main(String[] args) throws IOException { log.info("Connection to openBIS was successful."); log.info(String.format("%s provided openBIS identifiers have been found: %s", - identifiers.size(), identifiers.toString())); + commandLine.ids.size(), commandLine.ids.toString())); - for (String ident : identifiers) { + for (String ident : commandLine.ids) { log.info(String.format("Downloading files for provided identifier %s", ident)); List foundDataSets = qbicDataLoader.findAllDatasets(ident); diff --git a/src/main/java/life/qbic/QbicDataLoader.java b/src/main/java/life/qbic/QbicDataLoader.java index 4b818fe..157bce9 100644 --- a/src/main/java/life/qbic/QbicDataLoader.java +++ b/src/main/java/life/qbic/QbicDataLoader.java @@ -59,8 +59,8 @@ public class QbicDataLoader { */ public QbicDataLoader(String AppServerUri, String DataServerUri, String user, String password, - String bufferSize){ - this.defaultBufferSize = Integer.parseInt(bufferSize); + int bufferSize){ + this.defaultBufferSize = bufferSize; this.AppServerUri = AppServerUri; this.DataServerUri = DataServerUri; if (!AppServerUri.isEmpty()){ From ab7b03a414528bf89169c1ca917a0a4ba525bc4d Mon Sep 17 00:00:00 2001 From: sven1103 Date: Mon, 23 Apr 2018 16:38:32 +0200 Subject: [PATCH 03/15] Add new command line class --- src/main/java/life/qbic/App.java | 1 - src/main/java/life/qbic/MyCommandLine.java | 25 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/main/java/life/qbic/MyCommandLine.java diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index ac1ef27..69878fe 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -13,7 +13,6 @@ import picocli.CommandLine; - /** * qPostMan for staging data from openBIS * diff --git a/src/main/java/life/qbic/MyCommandLine.java b/src/main/java/life/qbic/MyCommandLine.java new file mode 100644 index 0000000..db0c5bc --- /dev/null +++ b/src/main/java/life/qbic/MyCommandLine.java @@ -0,0 +1,25 @@ +package life.qbic; + +import java.nio.file.Path; +import java.util.List; + +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +public class MyCommandLine { + + @Option(names = {"-u", "--user"}, required = true, description = "openBIS user name") + protected String user; + + @Parameters(paramLabel = "SAMPLE_ID", description = "one or more QBiC sample ids") + protected List ids; + + @Option(names = {"-f", "--file"}, description = "a file with line-separated list of QBiC sample ids") + protected Path filePath; + + @Option(names = {"-h", "--help"}, usageHelp = true, description = "display a help message") + protected boolean helpRequested = false; + + @Option(names = {"-b", "--buffer-size"}, description = "a integer muliple of 1024 bytes (default). Only change this if you know what you are doing.") + protected int bufferMultiplier = 1; +} \ No newline at end of file From 9db64b0d00f7fb7f8416e3f549c08e7df3e54810 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Mon, 23 Apr 2018 16:40:24 +0200 Subject: [PATCH 04/15] Change version to dev --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 659006d..760cc86 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ life.qbic qpostman jar - 0.1.3 + 0.1.3-SNAPSHOT qpostman http://maven.apache.org From f22d2b88909f0841623555d3171480f71087c2ea Mon Sep 17 00:00:00 2001 From: sven1103 Date: Mon, 23 Apr 2018 16:42:40 +0200 Subject: [PATCH 05/15] Refactor code --- pom.xml | 6 -- src/main/java/life/qbic/App.java | 5 -- src/main/java/life/qbic/Argparser.java | 80 +------------------------- 3 files changed, 1 insertion(+), 90 deletions(-) diff --git a/pom.xml b/pom.xml index 760cc86..f51dc2e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,12 +29,6 @@ openbis_api 3-S253.0 - - - commons-cli - commons-cli - 1.4 - org.apache.logging.log4j diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 69878fe..f1dd55d 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -29,11 +29,6 @@ public static void main(String[] args) throws IOException { MyCommandLine commandLine = new MyCommandLine(); new CommandLine(commandLine).parse(args); - if (commandLine.user == null) { - Argparser.printHelp(); - System.exit(1); - } - if ((commandLine.ids == null || commandLine.ids.isEmpty()) && (commandLine.filePath == null || commandLine.filePath == null)) { System.out .println("You have to provide one ID as command line argument or a file containing IDs."); diff --git a/src/main/java/life/qbic/Argparser.java b/src/main/java/life/qbic/Argparser.java index ee907aa..18015ca 100644 --- a/src/main/java/life/qbic/Argparser.java +++ b/src/main/java/life/qbic/Argparser.java @@ -4,90 +4,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Scanner; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - public class Argparser { - private static final HelpFormatter help = new HelpFormatter(); - - private static final Options options = new Options(); - - /** - * Public method for parsing the command line arguments - * - * @param args The args String array from cmd - * @return A hashmap with the parsed arguments - */ - public static HashMap parseCmdArguments(String[] args) { - - /* - * define the options - */ - options.addOption("h", "help", false, "Print this help"); - options.addOption("u", "user-name", true, "openBIS user name"); - options.addOption("i", "identifier", true, "openBis sample ID [mutually exclusive]"); - options.addOption("f", "file", true, - "File containing openBis sample IDs (one per line) [mutually exclusive]"); - options.addOption("b", "buffer-size", true, "A integer muliple of 1024 bytes (default). Only change this if you know what you are doing."); - - /* - * container for parsed attributes - */ - HashMap parsedArguments = new HashMap<>(); - - /* - * set parser type, default is enough here - */ - CommandLineParser parser = new DefaultParser(); - - /* - * try to parse the command line arguments - */ - try { - CommandLine cmd = parser.parse(options, args); - parsedArguments.put(Attribute.USERNAME, cmd.getOptionValue("u")); - parsedArguments.put(Attribute.ID, cmd.getOptionValue("i")); - parsedArguments.put(Attribute.FILE, cmd.getOptionValue("f")); - if (cmd.hasOption("h")) { - parsedArguments.put(Attribute.HELP, ""); - } - if (cmd.hasOption("b")){ - Integer mult = 1; - try{ - mult = Integer.parseInt(cmd.getOptionValue("b")); - } catch (Exception exc){ - exc.printStackTrace(); - System.err.println("buffer size must be an Integer > 0"); - } - parsedArguments.put(Attribute.BUFFER_SIZE, Integer.toString(mult * 1024)); - } else { - parsedArguments.put(Attribute.BUFFER_SIZE, Integer.toString(1024)); - } - } catch (ParseException exc) { - System.err.println(exc); - /* - * return empty map on fail - */ - return parsedArguments; - } - - return parsedArguments; - } - - public static void printHelp() { - help.printHelp("qPostMan", options, true); - } - - /** + /** * Retrieve the password from input stream * * @return The password From 3aa7079746632ea0160ceda920fee80877ecb7d4 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Mon, 23 Apr 2018 16:50:32 +0200 Subject: [PATCH 06/15] Show help message on -h --- src/main/java/life/qbic/App.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index f1dd55d..828eb89 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -29,6 +29,11 @@ public static void main(String[] args) throws IOException { MyCommandLine commandLine = new MyCommandLine(); new CommandLine(commandLine).parse(args); + if (commandLine.helpRequested){ + CommandLine.usage(new MyCommandLine(), System.out); + System.exit(0); + } + if ((commandLine.ids == null || commandLine.ids.isEmpty()) && (commandLine.filePath == null || commandLine.filePath == null)) { System.out .println("You have to provide one ID as command line argument or a file containing IDs."); From b028b90d498b787625b6dd4087aec20114349585 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 11:27:11 +0200 Subject: [PATCH 07/15] Change version to alpha --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f51dc2e..20903c9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ life.qbic qpostman jar - 0.1.3-SNAPSHOT + 0.1.3-alpha-SNAPSHOT qpostman http://maven.apache.org From f73e176313ffc7a7f482fca2a680a320d3f55f5c Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 13:07:40 +0200 Subject: [PATCH 08/15] Solves #32 --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20903c9..043f830 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ maven-dependency-plugin 3.1.0 + junit junit @@ -29,6 +30,13 @@ openbis_api 3-S253.0 + + + org.apache.httpcomponents + httpclient + 4.5.5 + + org.apache.logging.log4j @@ -38,7 +46,7 @@ jline jline - 2.14.2 + 2.14.6 From 81ad45b0b25db1b06cd881a6cf712cf4e0ccbc59 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 13:16:35 +0200 Subject: [PATCH 09/15] Print usage message, when no option is given --- src/main/java/life/qbic/App.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 828eb89..73bd48f 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -26,6 +26,11 @@ public class App { public static void main(String[] args) throws IOException { + if (args.length == 0){ + CommandLine.usage(new MyCommandLine(), System.out); + System.exit(0); + } + MyCommandLine commandLine = new MyCommandLine(); new CommandLine(commandLine).parse(args); From 6a20d8a2532f07803e834587d531a9a0884521cf Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 13:30:18 +0200 Subject: [PATCH 10/15] Introduce data-set type flag --- src/main/java/life/qbic/MyCommandLine.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/life/qbic/MyCommandLine.java b/src/main/java/life/qbic/MyCommandLine.java index db0c5bc..edefa53 100644 --- a/src/main/java/life/qbic/MyCommandLine.java +++ b/src/main/java/life/qbic/MyCommandLine.java @@ -22,4 +22,8 @@ public class MyCommandLine { @Option(names = {"-b", "--buffer-size"}, description = "a integer muliple of 1024 bytes (default). Only change this if you know what you are doing.") protected int bufferMultiplier = 1; + + @Option(names = {"-t", "--type"}, description = "filter for a given openBIS dataset type") + protected String datasetType = null; + } \ No newline at end of file From eca7f8a05730ae4053070f537992b3778b4bf226 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 13:50:42 +0200 Subject: [PATCH 11/15] Fetch the Dataset type and download only filtered --- src/main/java/life/qbic/App.java | 3 ++- src/main/java/life/qbic/MyCommandLine.java | 2 +- src/main/java/life/qbic/QbicDataLoader.java | 22 ++++++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 73bd48f..0e0e9db 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -60,7 +60,8 @@ public static void main(String[] args) throws IOException { System.exit(1); } - QbicDataLoader qbicDataLoader = new QbicDataLoader(AS_URL, DSS_URL, commandLine.user, password, commandLine.bufferMultiplier*1024); + QbicDataLoader qbicDataLoader = new QbicDataLoader(AS_URL, DSS_URL, commandLine.user, password, + commandLine.bufferMultiplier*1024, commandLine.datasetType); int returnCode = qbicDataLoader.login(); log.info(String.format("OpenBis login returned with %s", returnCode)); if (returnCode != 0) { diff --git a/src/main/java/life/qbic/MyCommandLine.java b/src/main/java/life/qbic/MyCommandLine.java index edefa53..5a59316 100644 --- a/src/main/java/life/qbic/MyCommandLine.java +++ b/src/main/java/life/qbic/MyCommandLine.java @@ -24,6 +24,6 @@ public class MyCommandLine { protected int bufferMultiplier = 1; @Option(names = {"-t", "--type"}, description = "filter for a given openBIS dataset type") - protected String datasetType = null; + protected String datasetType = ""; } \ No newline at end of file diff --git a/src/main/java/life/qbic/QbicDataLoader.java b/src/main/java/life/qbic/QbicDataLoader.java index 157bce9..bec2226 100644 --- a/src/main/java/life/qbic/QbicDataLoader.java +++ b/src/main/java/life/qbic/QbicDataLoader.java @@ -47,6 +47,8 @@ public class QbicDataLoader { private String sessionToken; + private String filterType; + private final int defaultBufferSize; /** @@ -59,10 +61,12 @@ public class QbicDataLoader { */ public QbicDataLoader(String AppServerUri, String DataServerUri, String user, String password, - int bufferSize){ + int bufferSize, String filterType){ this.defaultBufferSize = bufferSize; this.AppServerUri = AppServerUri; this.DataServerUri = DataServerUri; + this.filterType = filterType; + if (!AppServerUri.isEmpty()){ this.applicationServer = HttpInvokerUtils.createServiceStub( IApplicationServerApi.class, @@ -128,12 +132,13 @@ public List findAllDatasets(String sampleId) { // tell the API to fetch all descendents for each returned sample SampleFetchOptions fetchOptions = new SampleFetchOptions(); DataSetFetchOptions dsFetchOptions = new DataSetFetchOptions(); + dsFetchOptions.withType(); fetchOptions.withChildrenUsing(fetchOptions); fetchOptions.withDataSetsUsing(dsFetchOptions); SearchResult result = applicationServer.searchSamples(sessionToken, criteria, fetchOptions); // get all datasets of sample with provided sample code and all descendents - List foundDatasets = new ArrayList(); + List foundDatasets = new ArrayList<>(); for (Sample sample : result.getObjects()) { foundDatasets.addAll(sample.getDataSets()); for (Sample desc : sample.getChildren()) { @@ -141,7 +146,18 @@ public List findAllDatasets(String sampleId) { } } - return foundDatasets; + if (filterType.isEmpty()) + return foundDatasets; + + List filteredDatasets = new ArrayList<>(); + for (DataSet ds : foundDatasets){ + System.out.println(ds.getType().getCode() + " found."); + if (this.filterType.equals(ds.getType().getCode())){ + + filteredDatasets.add(ds); + } + } + return filteredDatasets; } /** From 3b6fcff257576474fb4777ff3e738f4088bff816 Mon Sep 17 00:00:00 2001 From: sven1103 Date: Tue, 24 Apr 2018 14:35:42 +0200 Subject: [PATCH 12/15] Update postman name and README --- README.md | 132 ++++++++++++++++++++++++++----- pom.xml | 6 +- src/main/java/life/qbic/App.java | 2 +- src/main/resources/log4j2.xml | 2 +- 4 files changed, 116 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b8a54ae..09bd9cc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![Build Status](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/qPostMan-development/badge/icon)](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/qPostMan-development/) +[![Build Status](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/postman-development/badge/icon)](https://qbic-intranet.am10.uni-tuebingen.de/jenkins/job/postman-development/) -# qPostMan +# postman **Current stable version: 0.1.2.3** @@ -9,11 +9,11 @@ A client software written in Java for dataset downloads from QBiC's data managem We are making use of the V3 API of openBIS (https://wiki-bsse.ethz.ch/display/openBISDoc1605/openBIS+V3+API) in order to interact with the data management system from command line, in order to provide a quick data retreaval on server or cluster resources, where the download via the qPortal is impractical. ## Download -You can download qPostMan from our repositoy with i.e. `wget`: +You can download postman from our repositoy with i.e. `wget`: ```bash VERSION=0.1.2.3 -wget https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases/life/qbic/qpostman/$VERSION/qpostman-$VERSION-jar-with-dependencies.jar -wget https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases/life/qbic/qpostman/$VERSION/qpostman-$VERSION-jar-with-dependencies.jar.md5 +wget https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases/life/qbic/postman/$VERSION/postman-$VERSION-jar-with-dependencies.jar +wget https://qbic-repo.am10.uni-tuebingen.de/repository/maven-releases/life/qbic/postman/$VERSION/postman-$VERSION-jar-with-dependencies.jar.md5 ``` Please compare the md5 checksum after the download. @@ -22,30 +22,32 @@ You need to have **Java JRE** or **JDK** installed (**openJDK** is fine), at lea ## Usage ### Options -Just execute qPostMan with `java -jar qpostman.jar` or `java -jar qpostman.jar -h` to get an overview of the options: +Just execute postman with `java -jar postman.jar` or `java -jar postman.jar -h` to get an overview of the options: ```bash -~$ java -jar qpostman.jar -usage: qPostMan [-b ] [-f ] [-h] [-i ] [-u ] - -b,--buffer-size A integer muliple of 1024 bytes (default). Only - change this if you know what you are doing. - -f,--file File containing openBis sample IDs (one per - line) [mutually exclusive] - -h,--help Print this help - -i,--identifier openBis sample ID [mutually exclusive] - -u,--user-name openBIS user name +~$ java -jar postman.jar +Usage:
[-h] [-b=] [-f=] + [-t=] -u= [SAMPLE_ID]... + [SAMPLE_ID]... one or more QBiC sample ids + -b, --buffer-size= + a integer muliple of 1024 bytes (default). Only + change this if you know what you are doing. + -f, --file= a file with line-separated list of QBiC sample ids + -h, --help display a help message + -t, --type= filter for a given openBIS dataset type + -u, --user= openBIS user name ``` ### Provide a QBiC ID The simplest scenario is, that you want to download a dataset/datasets from a sample. Just provide the QBiC ID for that sample and your username (same as the one you use for the qPortal): ```bash -~$ java -jar qpostman.jar -i -u +~$ java -jar postman.jar -i -u ``` -qPostMan will prompt you for your password, which is the password from your QBiC user account. +postman will prompt you for your password, which is the password from your QBiC user account. -After you have provided your password and authenticate successfully, qPostMan tries to download all datasets that are registered for that given sample ID and downloads them to the current working directory: +After you have provided your password and authenticate successfully, postman tries to download all datasets that are registered for that given sample ID and downloads them to the current working directory: ```bash -[bbbfs01@u-003-ncmu03 ~]$ qpostman-0.1.2.3 -i QMFKD003AG -u bbbfs01 +[bbbfs01@u-003-ncmu03 ~]$ java -jar postman.jar -u bbbfs01 QMFKD003AG Provide password for user 'bbbfs01': 12:32:02.038 [main] INFO life.qbic.App - OpenBis login returned with 0 @@ -57,13 +59,101 @@ Provide password for user 'bbbfs01': QMFKD003AG_SRR099967_1.filt.fastq.gz [### ] 0.38/7.94 Gb ``` +### Filter for dataset type + +You can filter for dataset types, using the `-t` option and one of the following openBIS dataset types we are currently using: + +```bash +ARR +AUDIT +CEL +CSV +EXPERIMENTAL_DESIGN +EXPRESSION_MATRIX +FASTQ +FEATUREXML +GZ +IDXML +JPG +MAT +MZML +PDF +PNG +Q_BMI_IMAGING_DATA +Q_DOCUMENT +Q_EXT_MS_QUALITYCONTROL_RESULTS +Q_EXT_NGS_QUALITYCONTROL_RESULTS +Q_FASTA_DATA +Q_HT_QPCR_DATA +Q_MA_AGILENT_DATA +Q_MA_CHIP_IMAGE +Q_MA_RAW_DATA +Q_MS_MZML_DATA +Q_MS_RAW_DATA +Q_MTB_ARCHIVE +Q_NGS_HLATYPING_DATA +Q_NGS_IMMUNE_MONITORING_DATA +Q_NGS_IONTORRENT_DATA +Q_NGS_MAPPING_DATA +Q_NGS_MTB_DATA +Q_NGS_RAW_DATA +Q_NGS_READ_MATCH_ARCHIVE +Q_NGS_VARIANT_CALLING_DATA +Q_PEPTIDE_DATA +Q_PROJECT_DATA +Q_TEST +Q_VACCINE_CONSTRUCT_DATA +Q_WF_EDDA_BENCHMARK_LOGS +Q_WF_EDDA_BENCHMARK_RESULTS +Q_WF_MA_QUALITYCONTROL_LOGS +Q_WF_MA_QUALITYCONTROL_RESULTS +Q_WF_MS_INDIVIDUALIZED_PROTEOME_LOGS +Q_WF_MS_INDIVIDUALIZED_PROTEOME_RESULTS +Q_WF_MS_LIGANDOMICS_ID_LOGS +Q_WF_MS_LIGANDOMICS_ID_RESULTS +Q_WF_MS_LIGANDOMICS_QC_LOGS +Q_WF_MS_LIGANDOMICS_QC_RESULTS +Q_WF_MS_MAXQUANT_LOGS +Q_WF_MS_MAXQUANT_ORIGINAL_OUT +Q_WF_MS_MAXQUANT_RESULTS +Q_WF_MS_PEAKPICKING_LOGS +Q_WF_MS_PEPTIDEID_LOGS +Q_WF_MS_PEPTIDEID_RESULTS +Q_WF_MS_QUALITYCONTROL_LOGS +Q_WF_MS_QUALITYCONTROL_RESULTS +Q_WF_NGS_16S_TAXONOMIC_PROFILING_LOGS +Q_WF_NGS_EPITOPE_PREDICTION_LOGS +Q_WF_NGS_EPITOPE_PREDICTION_RESULTS +Q_WF_NGS_HLATYPING_LOGS +Q_WF_NGS_HLATYPING_RESULTS +Q_WF_NGS_MAPPING_LOGS +Q_WF_NGS_MAPPING_RESULTS +Q_WF_NGS_QUALITYCONTROL_LOGS +Q_WF_NGS_QUALITYCONTROL_RESULTS +Q_WF_NGS_RNAEXPRESSIONANALYSIS_LOGS +Q_WF_NGS_RNAEXPRESSIONANALYSIS_RESULTS +Q_WF_NGS_SHRNA_COUNTING_LOGS +Q_WF_NGS_SHRNA_COUNTING_RESULTS +Q_WF_NGS_VARIANT_ANNOTATION_LOGS +Q_WF_NGS_VARIANT_ANNOTATION_RESULTS +Q_WF_NGS_VARIANT_CALLING_LOGS +Q_WF_NGS_VARIANT_CALLING_RESULTS +RAW +SHA256SUM +TAR +UNKNOWN +VCF +``` + ### Provide a file with several QBiC IDs -In order to download datasets from several samples at once, you can provide a simple text file with multiple, line-separated, QBiC IDs and hand it to qPostMan with the `-i` option. +In order to download datasets from several samples at once, you can provide a simple text file with multiple, line-separated, QBiC IDs and hand it to postman with the `-f` option. -qPostMan will automatically iterate over the IDs and try to download them. +postman will automatically iterate over the IDs and try to download them. ### Performance issues We discovered, that a default buffer size of 1024 bytes seems not always to get all out of the performance that is possible for the dataset download. Therefore, we allow you to enter a multipler Integer value that increases the buffer size. For example a multipler of 2 will result in 2x1024 = 2048 bytes and so on. Just use the `-b` option for that. The default buffer size remains 1024 bytes, if you don't specify this value. + + diff --git a/pom.xml b/pom.xml index 043f830..faee1ab 100644 --- a/pom.xml +++ b/pom.xml @@ -2,10 +2,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 life.qbic - qpostman + postman jar - 0.1.3-alpha-SNAPSHOT - qpostman + 0.1.3-alpha + postman http://maven.apache.org 1.8 diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 0e0e9db..37c6d67 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -14,7 +14,7 @@ /** - * qPostMan for staging data from openBIS + * postman for staging data from openBIS * */ public class App { diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 2acb93a..750d7d2 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -4,7 +4,7 @@ - + From 2470847eccb4ac7cb165b618af8a34281ea88497 Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 24 Apr 2018 16:26:49 +0200 Subject: [PATCH 13/15] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index faee1ab..4c364c2 100644 --- a/pom.xml +++ b/pom.xml @@ -1,8 +1,8 @@ 4.0.0 - life.qbic - postman + life.qbic.cli + postman-cli jar 0.1.3-alpha postman From 2662484dca83e105c66808d70c1c46cba60fdbaf Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 24 Apr 2018 16:40:15 +0200 Subject: [PATCH 14/15] Remove Assemply ID --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 4c364c2..0a4ec64 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,7 @@ jar-with-dependencies + false From beb79e173a42989131456fcc7b8c1b31004a9f1b Mon Sep 17 00:00:00 2001 From: Sven F Date: Mon, 2 Jul 2018 14:22:41 +0200 Subject: [PATCH 15/15] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a4ec64..024b085 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ life.qbic.cli postman-cli jar - 0.1.3-alpha + 0.1.3 postman http://maven.apache.org