D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

save files

This commit is contained in:
Michele Artini 2021-03-22 14:09:52 +01:00
parent b5ae4b92ab
commit 9bd0f1ae47
2 changed files with 129 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerClientApp.java
View File

@ -1,10 +1,7 @@
package eu.dnetlib.broker; package eu.dnetlib.broker;
import java.io.File;
import java.net.URL; import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.CommandLineParser;
@ -13,22 +10,15 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option; import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.configurationprocessor.json.JSONArray;
import org.springframework.boot.configurationprocessor.json.JSONException;
import org.springframework.boot.configurationprocessor.json.JSONObject;
@SpringBootApplication @SpringBootApplication
public class BrokerClientApp implements CommandLineRunner { public class BrokerClientApp implements CommandLineRunner {
@ -42,6 +32,9 @@ public class BrokerClientApp implements CommandLineRunner {
@Value("${dhp.broker.api.base-url}") @Value("${dhp.broker.api.base-url}")
private String defaultBrokerApiBaseUrl; private String defaultBrokerApiBaseUrl;
@Autowired
private BrokerUtils brokerUtils;
private final static CommandLineParser cmdLineParser = new DefaultParser(); private final static CommandLineParser cmdLineParser = new DefaultParser();
private final static Options options = new Options() private final static Options options = new Options()
@ -116,61 +109,37 @@ public class BrokerClientApp implements CommandLineRunner {
final String user = cmd.getOptionValue("u"); final String user = cmd.getOptionValue("u");
final URL baseUrl = new URL(cmd.getOptionValue("bu", defaultBrokerApiBaseUrl)); final URL baseUrl = new URL(cmd.getOptionValue("bu", defaultBrokerApiBaseUrl));
final String outputDir = cmd.getOptionValue("o"); final File outputDir = prepareDir(cmd.getOptionValue("o"));
log.info("* PARAMS: USER: " + user); log.info("* PARAMS: USER: " + user);
log.info("* PARAMS: BASE_URL: " + baseUrl); log.info("* PARAMS: BASE_URL: " + baseUrl);
log.info("* PARAMS: OUPUT DIR: " + outputDir); log.info("* PARAMS: OUPUT DIR: " + outputDir);
listSubscriptions(baseUrl, user); for (final String s : brokerUtils.listSubscriptions(baseUrl, user)) {
brokerUtils.downloadEvents(baseUrl, s, outputDir);
}
log.info("**** DONE ***"); log.info("**** DONE ***");
System.out.println(); System.out.println();
} }
private List<String> listSubscriptions(final URL baseUrl, final String email) throws Exception { private File prepareDir(final String path) {
final String url = baseUrl + "/subscriptions?email=" + URLEncoder.encode(email, StandardCharsets.UTF_8.name()); final File dir = new File(path);
log.info("Performing HTTP GET for subscriptions: " + url);
final HttpClient client = HttpClientBuilder.create().build(); if (dir.exists() && dir.isDirectory()) {
final HttpGet request = new HttpGet(url); log.info("Reusing existent directory: " + path);
request.addHeader("accept", "application/json"); return dir;
final HttpResponse response = client.execute(request);
final String json = IOUtils.toString(response.getEntity().getContent());
log.info("Found subscriptions: " + json);
final JSONArray array = new JSONArray(json);
System.out.println(String.format("Found %d subscription(s):", array.length()));
final List<String> res = new ArrayList<>();
for (int i = 0; i < array.length(); i++) {
final JSONObject object = array.getJSONObject(i);
final String suscrId = object.getString("subscriptionId");
final String topic = object.getString("topic");
final String ds = extractDsName(object.getJSONArray("conditionsAsList"));
System.out.println(String.format(" - %s (TOPIC: %s, DS: %s)", suscrId, topic, ds));
res.add(suscrId);
} }
return res; if (!dir.exists() && dir.mkdirs()) {
} log.info("New directory created: " + path);
return dir;
private String extractDsName(final JSONArray conds) {
try {
for (int i = 0; i < conds.length(); i++) {
final JSONObject object = conds.getJSONObject(i);
if (object.getString("field").equals("targetDatasourceName")) { return object.getJSONArray("listParams").getJSONObject(0).getString("value"); }
}
} catch (final JSONException e) {
log.warn(e.getMessage());
} }
return ""; log.error("Invalid directory: " + path);
throw new RuntimeException("Invalid directory: " + path);
} }
private static void printHelpAndExit(final Options options) { private static void printHelpAndExit(final Options options) {

110
cmd-line-apps/dhp-broker-client/src/main/java/eu/dnetlib/broker/BrokerUtils.java Normal file
View File

@ -0,0 +1,110 @@
package eu.dnetlib.broker;
import java.io.File;
import java.io.FileWriter;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.configurationprocessor.json.JSONArray;
import org.springframework.boot.configurationprocessor.json.JSONException;
import org.springframework.boot.configurationprocessor.json.JSONObject;
import org.springframework.stereotype.Component;
@Component
public class BrokerUtils {
private static final Logger log = LoggerFactory.getLogger(BrokerUtils.class);
private static final HttpClient client = HttpClientBuilder.create().build();
public List<String> listSubscriptions(final URL baseUrl, final String email) throws Exception {
final String url = baseUrl + "/subscriptions?email=" + URLEncoder.encode(email, StandardCharsets.UTF_8.name());
log.info("Performing HTTP GET for subscriptions: " + url);
final HttpGet request = new HttpGet(url);
request.addHeader("accept", "application/json");
final HttpResponse response = client.execute(request);
final String json = IOUtils.toString(response.getEntity().getContent());
log.info("Found subscriptions: " + json);
final JSONArray array = new JSONArray(json);
System.out.println(String.format("Found %d subscription(s):", array.length()));
final List<String> res = new ArrayList<>();
for (int i = 0; i < array.length(); i++) {
final JSONObject object = array.getJSONObject(i);
final String suscrId = object.getString("subscriptionId");
final String topic = object.getString("topic");
final String ds = extractDsName(object.getJSONArray("conditionsAsList"));
System.out.println(String.format(" - %s (TOPIC: %s, DS: %s)", suscrId, topic, ds));
res.add(suscrId);
}
System.out.println();
return res;
}
public void downloadEvents(final URL baseUrl, final String subscrId, final File outputDir) throws Exception {
final String fp = String.format("%s/%s.json", outputDir.getAbsolutePath(), subscrId);
System.out.print("Saving file " + fp + ": ");
System.out.flush();
try (final FileWriter file = new FileWriter(fp)) {
String url = baseUrl + "/scroll/notifications/bySubscriptionId/" + URLEncoder.encode(subscrId, StandardCharsets.UTF_8.name());
boolean notCompleted = false;
do {
log.info("Performing HTTP GET for notifications: " + url);
final HttpGet request = new HttpGet(url);
request.addHeader("accept", "application/json");
final HttpResponse response = client.execute(request);
final String json = IOUtils.toString(response.getEntity().getContent());
final JSONObject data = new JSONObject(json);
final JSONArray values = data.getJSONArray("values");
for (int i = 0; i < values.length(); i++) {
file.append(values.getJSONObject(i).toString());
file.append("\n");
}
notCompleted = !data.getBoolean("completed");
url = baseUrl + "/scroll/notifications/" + data.getString("id");
System.out.print(".");
System.out.flush();
} while (notCompleted);
System.out.println();
}
}
private String extractDsName(final JSONArray conds) {
try {
for (int i = 0; i < conds.length(); i++) {
final JSONObject object = conds.getJSONObject(i);
if (object.getString("field").equals("targetDatasourceName")) { return object.getJSONArray("listParams").getJSONObject(0).getString("value"); }
}
} catch (final JSONException e) {
log.warn(e.getMessage());
}
return "";
}
}