From 6856a07ed53846fb73ac325bcb320defaf7593db Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 23 Mar 2021 09:45:16 +0100 Subject: [PATCH] gzip and interactive mode --- .../eu/dnetlib/broker/BrokerClientApp.java | 37 +++++++++- .../java/eu/dnetlib/broker/BrokerUtils.java | 71 +++++++++++-------- 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java b/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java index d0885620..4bfcdadc 100644 --- a/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java +++ b/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java @@ -1,6 +1,7 @@ package eu.dnetlib.broker; import java.io.File; +import java.io.IOException; import java.net.URL; import org.apache.commons.cli.CommandLine; @@ -56,6 +57,16 @@ public class BrokerClientApp implements CommandLineRunner { .longOpt("output") .desc("the output dir (REQUIRED)") .build()) + .addOption(Option.builder("z") + .required(false) + .hasArg(false) + .desc("compress the output files in GZIP format") + .build()) + .addOption(Option.builder("i") + .required(false) + .hasArg(false) + .desc("interactive mode") + .build()) .addOption(Option.builder("h") .longOpt("help") .required(false) @@ -110,13 +121,19 @@ public class BrokerClientApp implements CommandLineRunner { final String user = cmd.getOptionValue("u"); final URL baseUrl = new URL(cmd.getOptionValue("bu", defaultBrokerApiBaseUrl)); final File outputDir = prepareDir(cmd.getOptionValue("o")); + final boolean gzip = cmd.hasOption("z"); + final boolean interactive = cmd.hasOption("i"); log.info("* PARAMS: USER: " + user); log.info("* PARAMS: BASE_URL: " + baseUrl); log.info("* PARAMS: OUPUT DIR: " + outputDir); + log.info("* PARAMS: SAVE AS GZIP: " + gzip); + log.info("* PARAMS: INTERACTIVE MODE: " + interactive); for (final String s : brokerUtils.listSubscriptions(baseUrl, user)) { - brokerUtils.downloadEvents(baseUrl, s, outputDir); + if (!interactive || confirm(s)) { + brokerUtils.downloadEvents(baseUrl, s, outputDir, gzip); + } } log.info("**** DONE ***"); @@ -124,6 +141,24 @@ public class BrokerClientApp implements CommandLineRunner { } + private boolean confirm(final String s) throws IOException { + System.out.print(String.format("Do you want download subscription %s? (Y/N) ", s)); + System.out.flush(); + + char c = (char) System.in.read(); + while (true) { + if (c == 'y' || c == 'Y') { + System.out.println(); + return true; + } else if (c == 'n' || c == 'N') { + System.out.println(); + return false; + } else { + c = (char) System.in.read(); + } + } + } + private File prepareDir(final String path) { final File dir = new File(path); diff --git a/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java b/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java index 65d52755..1fc7d079 100644 --- a/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java +++ b/cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java @@ -1,12 +1,16 @@ package eu.dnetlib.broker; import java.io.File; +import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.zip.GZIPOutputStream; import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; @@ -58,40 +62,49 @@ public class BrokerUtils { return res; } - public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir) throws Exception { - final String fp = String.format("%s/%s.json", outputDir.getAbsolutePath(), subscrId); + public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir, final boolean gzip) throws Exception { + final String fp = String.format(gzip ? "%s/%s.json.gz" : "%s/%s.json", outputDir.getAbsolutePath(), subscrId); System.out.print("Saving file " + fp + ": "); System.out.flush(); - try (final FileWriter file = new FileWriter(fp)) { - - String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name()); - - boolean notCompleted = false; - - do { - log.info("Performing HTTP GET for notifications: " + url); - final HttpGet request = new HttpGet(url); - request.addHeader("accept", "application/json"); - final HttpResponse response = client.execute(request); - final String json = IOUtils.toString(response.getEntity().getContent()); - final JSONObject data = new JSONObject(json); - - final JSONArray values = data.getJSONArray("values"); - for (int i = 0; i < values.length(); i++) { - file.append(values.getJSONObject(i).toString()); - file.append("\n"); - } - - notCompleted = !data.getBoolean("completed"); - url = baseUrl + "/scroll/notifications/" + data.getString("id"); - - System.out.print("."); - System.out.flush(); - } while (notCompleted); - System.out.println(); + if (gzip) { + try (final FileOutputStream fos = new FileOutputStream(fp); final Writer w = new OutputStreamWriter(new GZIPOutputStream(fos), "UTF-8")) { + writeEvents(baseUrl, subscrId, w); + } + } else { + try (final FileWriter fw = new FileWriter(fp)) { + writeEvents(baseUrl, subscrId, fw); + } } + System.out.println(); + } + + private void writeEvents(final URL baseUrl, final String subscrId, final Writer file) throws Exception { + + String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name()); + boolean notCompleted = false; + + do { + log.info("Performing HTTP GET for notifications: " + url); + final HttpGet request = new HttpGet(url); + request.addHeader("accept", "application/json"); + final HttpResponse response = client.execute(request); + final String json = IOUtils.toString(response.getEntity().getContent()); + final JSONObject data = new JSONObject(json); + + final JSONArray values = data.getJSONArray("values"); + for (int i = 0; i < values.length(); i++) { + file.append(values.getJSONObject(i).toString()); + file.append("\n"); + } + + notCompleted = !data.getBoolean("completed"); + url = baseUrl + "/scroll/notifications/" + data.getString("id"); + + System.out.print("."); + System.out.flush(); + } while (notCompleted); } private String extractDsName(final JSONArray conds) {