D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

gzip and interactive mode

This commit is contained in:
Michele Artini 2021-03-23 09:45:16 +01:00
parent 9bd0f1ae47
commit 6856a07ed5
2 changed files with 78 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.broker;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import org.apache.commons.cli.CommandLine;
@ -56,6 +57,16 @@ public class BrokerClientApp implements CommandLineRunner {
.longOpt("output")
.desc("the output dir (REQUIRED)")
.build())
.addOption(Option.builder("z")
.required(false)
.hasArg(false)
.desc("compress the output files in GZIP format")
.build())
.addOption(Option.builder("i")
.required(false)
.hasArg(false)
.desc("interactive mode")
.build())
.addOption(Option.builder("h")
.longOpt("help")
.required(false)
@ -110,13 +121,19 @@ public class BrokerClientApp implements CommandLineRunner {
final String user = cmd.getOptionValue("u");
final URL baseUrl = new URL(cmd.getOptionValue("bu", defaultBrokerApiBaseUrl));
final File outputDir = prepareDir(cmd.getOptionValue("o"));
final boolean gzip = cmd.hasOption("z");
final boolean interactive = cmd.hasOption("i");
log.info("* PARAMS: USER: " + user);
log.info("* PARAMS: BASE_URL: " + baseUrl);
log.info("* PARAMS: OUPUT DIR: " + outputDir);
log.info("* PARAMS: SAVE AS GZIP: " + gzip);
log.info("* PARAMS: INTERACTIVE MODE: " + interactive);
for (final String s : brokerUtils.listSubscriptions(baseUrl, user)) {
brokerUtils.downloadEvents(baseUrl, s, outputDir);
if (!interactive || confirm(s)) {
brokerUtils.downloadEvents(baseUrl, s, outputDir, gzip);
}
}
log.info("**** DONE ***");
@ -124,6 +141,24 @@ public class BrokerClientApp implements CommandLineRunner {
}
private boolean confirm(final String s) throws IOException {
System.out.print(String.format("Do you want download subscription %s? (Y/N) ", s));
System.out.flush();
char c = (char) System.in.read();
while (true) {
if (c == 'y' || c == 'Y') {
System.out.println();
return true;
} else if (c == 'n' || c == 'N') {
System.out.println();
return false;
} else {
c = (char) System.in.read();
}
}
}
private File prepareDir(final String path) {
final File dir = new File(path);

71
cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java
View File

@ -1,12 +1,16 @@
package eu.dnetlib.broker;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
@ -58,40 +62,49 @@ public class BrokerUtils {
return res;
}
public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir) throws Exception {
final String fp = String.format("%s/%s.json", outputDir.getAbsolutePath(), subscrId);
public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir, final boolean gzip) throws Exception {
final String fp = String.format(gzip ? "%s/%s.json.gz" : "%s/%s.json", outputDir.getAbsolutePath(), subscrId);
System.out.print("Saving file " + fp + ": ");
System.out.flush();
try (final FileWriter file = new FileWriter(fp)) {
String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name());
boolean notCompleted = false;
do {
log.info("Performing HTTP GET for notifications: " + url);
final HttpGet request = new HttpGet(url);
request.addHeader("accept", "application/json");
final HttpResponse response = client.execute(request);
final String json = IOUtils.toString(response.getEntity().getContent());
final JSONObject data = new JSONObject(json);
final JSONArray values = data.getJSONArray("values");
for (int i = 0; i < values.length(); i++) {
file.append(values.getJSONObject(i).toString());
file.append("\n");
}
notCompleted = !data.getBoolean("completed");
url = baseUrl + "/scroll/notifications/" + data.getString("id");
System.out.print(".");
System.out.flush();
} while (notCompleted);
System.out.println();
if (gzip) {
try (final FileOutputStream fos = new FileOutputStream(fp); final Writer w = new OutputStreamWriter(new GZIPOutputStream(fos), "UTF-8")) {
writeEvents(baseUrl, subscrId, w);
}
} else {
try (final FileWriter fw = new FileWriter(fp)) {
writeEvents(baseUrl, subscrId, fw);
}
}
System.out.println();
}
private void writeEvents(final URL baseUrl, final String subscrId, final Writer file) throws Exception {
String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name());
boolean notCompleted = false;
do {
log.info("Performing HTTP GET for notifications: " + url);
final HttpGet request = new HttpGet(url);
request.addHeader("accept", "application/json");
final HttpResponse response = client.execute(request);
final String json = IOUtils.toString(response.getEntity().getContent());
final JSONObject data = new JSONObject(json);
final JSONArray values = data.getJSONArray("values");
for (int i = 0; i < values.length(); i++) {
file.append(values.getJSONObject(i).toString());
file.append("\n");
}
notCompleted = !data.getBoolean("completed");
url = baseUrl + "/scroll/notifications/" + data.getString("id");
System.out.print(".");
System.out.flush();
} while (notCompleted);
}
private String extractDsName(final JSONArray conds) {