gzip and interactive mode
This commit is contained in:
parent
9bd0f1ae47
commit
6856a07ed5
|
@ -1,6 +1,7 @@
|
|||
package eu.dnetlib.broker;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
|
@ -56,6 +57,16 @@ public class BrokerClientApp implements CommandLineRunner {
|
|||
.longOpt("output")
|
||||
.desc("the output dir (REQUIRED)")
|
||||
.build())
|
||||
.addOption(Option.builder("z")
|
||||
.required(false)
|
||||
.hasArg(false)
|
||||
.desc("compress the output files in GZIP format")
|
||||
.build())
|
||||
.addOption(Option.builder("i")
|
||||
.required(false)
|
||||
.hasArg(false)
|
||||
.desc("interactive mode")
|
||||
.build())
|
||||
.addOption(Option.builder("h")
|
||||
.longOpt("help")
|
||||
.required(false)
|
||||
|
@ -110,13 +121,19 @@ public class BrokerClientApp implements CommandLineRunner {
|
|||
final String user = cmd.getOptionValue("u");
|
||||
final URL baseUrl = new URL(cmd.getOptionValue("bu", defaultBrokerApiBaseUrl));
|
||||
final File outputDir = prepareDir(cmd.getOptionValue("o"));
|
||||
final boolean gzip = cmd.hasOption("z");
|
||||
final boolean interactive = cmd.hasOption("i");
|
||||
|
||||
log.info("* PARAMS: USER: " + user);
|
||||
log.info("* PARAMS: BASE_URL: " + baseUrl);
|
||||
log.info("* PARAMS: OUPUT DIR: " + outputDir);
|
||||
log.info("* PARAMS: SAVE AS GZIP: " + gzip);
|
||||
log.info("* PARAMS: INTERACTIVE MODE: " + interactive);
|
||||
|
||||
for (final String s : brokerUtils.listSubscriptions(baseUrl, user)) {
|
||||
brokerUtils.downloadEvents(baseUrl, s, outputDir);
|
||||
if (!interactive || confirm(s)) {
|
||||
brokerUtils.downloadEvents(baseUrl, s, outputDir, gzip);
|
||||
}
|
||||
}
|
||||
|
||||
log.info("**** DONE ***");
|
||||
|
@ -124,6 +141,24 @@ public class BrokerClientApp implements CommandLineRunner {
|
|||
|
||||
}
|
||||
|
||||
private boolean confirm(final String s) throws IOException {
|
||||
System.out.print(String.format("Do you want download subscription %s? (Y/N) ", s));
|
||||
System.out.flush();
|
||||
|
||||
char c = (char) System.in.read();
|
||||
while (true) {
|
||||
if (c == 'y' || c == 'Y') {
|
||||
System.out.println();
|
||||
return true;
|
||||
} else if (c == 'n' || c == 'N') {
|
||||
System.out.println();
|
||||
return false;
|
||||
} else {
|
||||
c = (char) System.in.read();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private File prepareDir(final String path) {
|
||||
final File dir = new File(path);
|
||||
|
||||
|
|
|
@ -1,12 +1,16 @@
|
|||
package eu.dnetlib.broker;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpResponse;
|
||||
|
@ -58,40 +62,49 @@ public class BrokerUtils {
|
|||
return res;
|
||||
}
|
||||
|
||||
public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir) throws Exception {
|
||||
final String fp = String.format("%s/%s.json", outputDir.getAbsolutePath(), subscrId);
|
||||
public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir, final boolean gzip) throws Exception {
|
||||
final String fp = String.format(gzip ? "%s/%s.json.gz" : "%s/%s.json", outputDir.getAbsolutePath(), subscrId);
|
||||
|
||||
System.out.print("Saving file " + fp + ": ");
|
||||
System.out.flush();
|
||||
|
||||
try (final FileWriter file = new FileWriter(fp)) {
|
||||
|
||||
String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name());
|
||||
|
||||
boolean notCompleted = false;
|
||||
|
||||
do {
|
||||
log.info("Performing HTTP GET for notifications: " + url);
|
||||
final HttpGet request = new HttpGet(url);
|
||||
request.addHeader("accept", "application/json");
|
||||
final HttpResponse response = client.execute(request);
|
||||
final String json = IOUtils.toString(response.getEntity().getContent());
|
||||
final JSONObject data = new JSONObject(json);
|
||||
|
||||
final JSONArray values = data.getJSONArray("values");
|
||||
for (int i = 0; i < values.length(); i++) {
|
||||
file.append(values.getJSONObject(i).toString());
|
||||
file.append("\n");
|
||||
}
|
||||
|
||||
notCompleted = !data.getBoolean("completed");
|
||||
url = baseUrl + "/scroll/notifications/" + data.getString("id");
|
||||
|
||||
System.out.print(".");
|
||||
System.out.flush();
|
||||
} while (notCompleted);
|
||||
System.out.println();
|
||||
if (gzip) {
|
||||
try (final FileOutputStream fos = new FileOutputStream(fp); final Writer w = new OutputStreamWriter(new GZIPOutputStream(fos), "UTF-8")) {
|
||||
writeEvents(baseUrl, subscrId, w);
|
||||
}
|
||||
} else {
|
||||
try (final FileWriter fw = new FileWriter(fp)) {
|
||||
writeEvents(baseUrl, subscrId, fw);
|
||||
}
|
||||
}
|
||||
System.out.println();
|
||||
}
|
||||
|
||||
private void writeEvents(final URL baseUrl, final String subscrId, final Writer file) throws Exception {
|
||||
|
||||
String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name());
|
||||
boolean notCompleted = false;
|
||||
|
||||
do {
|
||||
log.info("Performing HTTP GET for notifications: " + url);
|
||||
final HttpGet request = new HttpGet(url);
|
||||
request.addHeader("accept", "application/json");
|
||||
final HttpResponse response = client.execute(request);
|
||||
final String json = IOUtils.toString(response.getEntity().getContent());
|
||||
final JSONObject data = new JSONObject(json);
|
||||
|
||||
final JSONArray values = data.getJSONArray("values");
|
||||
for (int i = 0; i < values.length(); i++) {
|
||||
file.append(values.getJSONObject(i).toString());
|
||||
file.append("\n");
|
||||
}
|
||||
|
||||
notCompleted = !data.getBoolean("completed");
|
||||
url = baseUrl + "/scroll/notifications/" + data.getString("id");
|
||||
|
||||
System.out.print(".");
|
||||
System.out.flush();
|
||||
} while (notCompleted);
|
||||
}
|
||||
|
||||
private String extractDsName(final JSONArray conds) {
|
||||
|
|
Loading…
Reference in New Issue