From 669a509430bb309399e1054b67028cb4d924240b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 19 Jun 2020 17:39:46 +0200 Subject: [PATCH] - --- .../dnetlib/dhp/oa/graph/dump/APIClient.java | 33 +++-- .../oa/graph/dump/QueryInformationSystem.java | 2 +- .../dhp/oa/graph/dump/SendToZenodo.java | 128 +++++++++--------- .../dhp/oa/graph/dump/upload_zenodo.json | 0 4 files changed, 85 insertions(+), 78 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java index ce68454ba..ae046c941 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java @@ -5,21 +5,12 @@ import java.io.*; import java.io.IOException; import java.nio.charset.StandardCharsets; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; - -import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.StringEntity; - import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; @@ -28,6 +19,8 @@ import com.google.gson.Gson; import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel; +//import org.apache.http.entity.mime.MultipartEntityBuilder; + public class APIClient implements Serializable { String urlString; @@ -81,9 +74,26 @@ public class APIClient implements Serializable { return response.getStatusLine().getStatusCode(); - } +// public int upload(InputStream is, String file_name) throws IOException { +// HttpClient client = new DefaultHttpClient(); +// +// HttpPut put = new HttpPut(bucket + "/" + file_name); +// put.setHeader("Authorization", "Bearer " + access_token); +// put.addHeader("Content-Type", "application/zip"); +// +// HttpEntity data = MultipartEntityBuilder +// .create() +// // .addPart("file", new ByteArrayInputStream(is)); +// .addBinaryBody(file_name, is, ContentType.APPLICATION_OCTET_STREAM, file_name) +// .build(); +// put.setEntity(data); +// +// HttpResponse response = client.execute(put); +// +// return response.getStatusLine().getStatusCode(); +// } public int upload(File file, String file_name) throws IOException { HttpClient client = new DefaultHttpClient(); @@ -112,12 +122,11 @@ public class APIClient implements Serializable { HttpResponse response = client.execute(post); return response.getStatusLine().getStatusCode(); - } public int publish() throws IOException { HttpClient client = new DefaultHttpClient(); - HttpPost post = new HttpPost(urlString +"/"+ deposition_id +"/actions/publish") ; + HttpPost post = new HttpPost(urlString + "/" + deposition_id + "/actions/publish"); post.setHeader("Authorization", "Bearer " + access_token); HttpResponse response = client.execute(post); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java index 4ff54eee1..d89934ac4 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/QueryInformationSystem.java @@ -40,7 +40,7 @@ public class QueryInformationSystem { this.isLookUp = isLookUpService; } - public static CommunityMap getMap(List communityMap) { + private CommunityMap getMap(List communityMap) { final CommunityMap map = new CommunityMap(); communityMap.stream().forEach(xml -> { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java index f9fecf505..64cbe41c0 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java @@ -1,14 +1,11 @@ + package eu.dnetlib.dhp.oa.graph.dump; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.DbClient; -import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator; -import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata; -import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.Relation; +import java.io.File; +import java.io.Serializable; + +import javax.management.Query; + import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -18,72 +15,73 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import java.io.BufferedWriter; -import java.io.File; -import java.io.Serializable; -import java.sql.ResultSet; -import java.util.Arrays; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Function; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; public class SendToZenodo implements Serializable { - private static final Log log = LogFactory.getLog(SendToZenodo.class); + private static final Log log = LogFactory.getLog(SendToZenodo.class); + public static void main(final String[] args) throws Exception { + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SendToZenodo.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json"))); - public static void main(final String[] args) throws Exception { - final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString( - SendToZenodo.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json"))); + parser.parseArgument(args); - parser.parseArgument(args); + final String hdfsPath = parser.get("hdfsPath"); + final String hdfsNameNode = parser.get("hdfsNameNode"); + final String access_token = parser.get("accessToken"); + final String connection_url = parser.get("connectionUrl"); + final String metadata = parser.get("metadata"); + final String isLookUpUrl = parser.get("isLookUpUrl"); + QueryInformationSystem qis = new QueryInformationSystem(); + qis.setIsLookUp(ISLookupClientFactory.getLookUpService(isLookUpUrl)); + CommunityMap communityMap = qis.getCommunityMap(); - final String hdfsPath = parser.get("hdfsPath"); - final String hdfsNameNode = parser.get("hdfsNameNode"); - final String access_token = parser.get("accessToken"); - final String connection_url = parser.get("url"); - final String metadata = parser.get("metadata"); - Configuration conf = new Configuration(); - conf.set("fs.defaultFS", hdfsNameNode); + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", hdfsNameNode); - FileSystem fileSystem = FileSystem.get(conf); + FileSystem fileSystem = FileSystem.get(conf); + RemoteIterator fileStatusListIterator = fileSystem + .listFiles( + new Path(hdfsPath), true); + APIClient apiClient = new APIClient(connection_url, access_token); + apiClient.connect(); + while (fileStatusListIterator.hasNext()) { + LocatedFileStatus fileStatus = fileStatusListIterator.next(); - RemoteIterator fileStatusListIterator = fileSystem.listFiles( - new Path(hdfsPath), true); - APIClient apiClient = new APIClient(connection_url, access_token); - apiClient.connect(); - while(fileStatusListIterator.hasNext()){ - LocatedFileStatus fileStatus = fileStatusListIterator.next(); + Path p = fileStatus.getPath(); + String p_string = p.toString(); + String tmp = p_string.substring(0, p_string.lastIndexOf("/")); + String community = tmp.substring(tmp.lastIndexOf("/") + 1); + log.info("Sending information for community: " + community); + String community_name = communityMap.get(community).replace(" ", "_"); + log.info("Copying information for community: " + community); + fileSystem.copyToLocalFile(p, new Path("/tmp/" + community_name)); + File f = new File("/tmp/" + community_name); + try { + apiClient.upload(f, community_name); + apiClient.sendMretadata(metadata); + apiClient.publish(); + } catch (Exception e) { + if (f.exists()) { + log.info("Deleting information for community: " + community); + f.delete(); + } + } finally { + if (f.exists()) { + log.info("Deleting information for community: " + community); + f.delete(); + } + } + } - Path p = fileStatus.getPath(); - String p_string = p.toString(); - String tmp = p_string.substring(0, p_string.lastIndexOf("/") ); - String community = tmp.substring(tmp.lastIndexOf("/") + 1); - log.info("Sending information for community: " + community); - fileSystem.copyToLocalFile(p, new Path("/tmp/" + community)); + } - - File f = new File("/tmp/" + community); - apiClient.upload(f, community); - apiClient.sendMretadata(metadata); - apiClient.publish(); - - if (f.exists()){ - f.delete(); - } - } - - - - - - } - - -} \ No newline at end of file +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json new file mode 100644 index 000000000..e69de29bb