From 44a12d244f5efaf84ddee6b9f73f92d61b28c07f Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 18 Jun 2020 18:38:54 +0200 Subject: [PATCH] - --- .../dnetlib/dhp/oa/graph/dump/APIClient.java | 40 +++++---- .../dhp/oa/graph/dump/SendToZenodo.java | 89 +++++++++++++++++++ 2 files changed, 111 insertions(+), 18 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java index 74d02aacf..ce68454ba 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java @@ -5,12 +5,19 @@ import java.io.*; import java.io.IOException; import java.nio.charset.StandardCharsets; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; @@ -27,7 +34,7 @@ public class APIClient implements Serializable { String bucket; String deposition_id; - final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe"; + String access_token; public String getUrlString() { return urlString; @@ -45,10 +52,10 @@ public class APIClient implements Serializable { this.bucket = bucket; } - public APIClient(String urlString) throws IOException { + public APIClient(String urlString, String access_token) throws IOException { this.urlString = urlString; - // connect(); + this.access_token = access_token; } public int connect() throws IOException { @@ -62,7 +69,7 @@ public class APIClient implements Serializable { StringEntity input = new StringEntity(json); post.setEntity(input); post.addHeader("Content-Type", "application/json"); - post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN); + post.setHeader("Authorization", "Bearer " + access_token); HttpResponse response = client.execute(post); @@ -77,48 +84,45 @@ public class APIClient implements Serializable { } - public void upload(String filePath, String file_name) throws IOException { + + public int upload(File file, String file_name) throws IOException { HttpClient client = new DefaultHttpClient(); - File file = new File(filePath); + HttpPut put = new HttpPut(bucket + "/" + file_name); - put.setHeader("Authorization", "Bearer " + ACCESS_TOKEN); + put.setHeader("Authorization", "Bearer " + access_token); put.addHeader("Content-Type", "application/zip"); HttpEntity data = MultipartEntityBuilder.create().addBinaryBody(file_name, file).build(); put.setEntity(data); HttpResponse response = client.execute(put); - String json = EntityUtils.toString(response.getEntity()); - - ZenodoModel newSubmission = new Gson().fromJson(json, ZenodoModel.class); - - System.out.println(response.getStatusLine().getStatusCode()); + return response.getStatusLine().getStatusCode(); } - public void sendMretadata(String metadata) throws IOException { + public int sendMretadata(String metadata) throws IOException { HttpClient client = new DefaultHttpClient(); HttpPut post = new HttpPut(urlString + "/" + deposition_id); - post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN); + post.setHeader("Authorization", "Bearer " + access_token); post.addHeader("Content-Type", "application/json"); StringEntity entity = new StringEntity(metadata, StandardCharsets.UTF_8); post.setEntity(entity); HttpResponse response = client.execute(post); - System.out.println(response.getStatusLine().getStatusCode()); + return response.getStatusLine().getStatusCode(); } - public void publish() throws IOException { + public int publish() throws IOException { HttpClient client = new DefaultHttpClient(); HttpPost post = new HttpPost(urlString +"/"+ deposition_id +"/actions/publish") ; - post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN); + post.setHeader("Authorization", "Bearer " + access_token); HttpResponse response = client.execute(post); - System.out.println(response.getStatusLine().getStatusCode()); + return response.getStatusLine().getStatusCode(); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java new file mode 100644 index 000000000..f9fecf505 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java @@ -0,0 +1,89 @@ +package eu.dnetlib.dhp.oa.graph.dump; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.DbClient; +import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator; +import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata; +import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Relation; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.Serializable; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +public class SendToZenodo implements Serializable { + + private static final Log log = LogFactory.getLog(SendToZenodo.class); + + + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SendToZenodo.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json"))); + + parser.parseArgument(args); + + + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + final String access_token = parser.get("accessToken"); + final String connection_url = parser.get("url"); + final String metadata = parser.get("metadata"); + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", hdfsNameNode); + + FileSystem fileSystem = FileSystem.get(conf); + + + RemoteIterator fileStatusListIterator = fileSystem.listFiles( + new Path(hdfsPath), true); + APIClient apiClient = new APIClient(connection_url, access_token); + apiClient.connect(); + while(fileStatusListIterator.hasNext()){ + LocatedFileStatus fileStatus = fileStatusListIterator.next(); + + Path p = fileStatus.getPath(); + String p_string = p.toString(); + String tmp = p_string.substring(0, p_string.lastIndexOf("/") ); + String community = tmp.substring(tmp.lastIndexOf("/") + 1); + log.info("Sending information for community: " + community); + fileSystem.copyToLocalFile(p, new Path("/tmp/" + community)); + + + File f = new File("/tmp/" + community); + apiClient.upload(f, community); + apiClient.sendMretadata(metadata); + apiClient.publish(); + + if (f.exists()){ + f.delete(); + } + } + + + + + + } + + +} \ No newline at end of file