-

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java

@@ -5,12 +5,19 @@ import java.io.*;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
 
 import org.apache.http.entity.mime.MultipartEntityBuilder;
@@ -27,7 +34,7 @@ public class APIClient implements Serializable {
 	String bucket;
 
 	String deposition_id;
-	final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe";
+	String access_token;
 
 	public String getUrlString() {
 		return urlString;
@@ -45,10 +52,10 @@ public class APIClient implements Serializable {
 		this.bucket = bucket;
 	}
 
-	public APIClient(String urlString) throws IOException {
+	public APIClient(String urlString, String access_token) throws IOException {
 
 		this.urlString = urlString;
-		// connect();
+		this.access_token = access_token;
 	}
 
 	public int connect() throws IOException {
@@ -62,7 +69,7 @@ public class APIClient implements Serializable {
 		StringEntity input = new StringEntity(json);
 		post.setEntity(input);
 		post.addHeader("Content-Type", "application/json");
-		post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
+		post.setHeader("Authorization", "Bearer " + access_token);
 
 		HttpResponse response = client.execute(post);
 
@@ -77,48 +84,45 @@ public class APIClient implements Serializable {
 
 	}
 
-	public void upload(String filePath, String file_name) throws IOException {
+
+	public int upload(File file, String file_name) throws IOException {
 		HttpClient client = new DefaultHttpClient();
-		File file = new File(filePath);
+
 		HttpPut put = new HttpPut(bucket + "/" + file_name);
-		put.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
+		put.setHeader("Authorization", "Bearer " + access_token);
 		put.addHeader("Content-Type", "application/zip");
 		HttpEntity data = MultipartEntityBuilder.create().addBinaryBody(file_name, file).build();
 		put.setEntity(data);
 
 		HttpResponse response = client.execute(put);
 
-		String json = EntityUtils.toString(response.getEntity());
+		return response.getStatusLine().getStatusCode();
-
-		ZenodoModel newSubmission = new Gson().fromJson(json, ZenodoModel.class);
-
-		System.out.println(response.getStatusLine().getStatusCode());
 
 	}
 
-	public void sendMretadata(String metadata) throws IOException {
+	public int sendMretadata(String metadata) throws IOException {
 
 		HttpClient client = new DefaultHttpClient();
 		HttpPut post = new HttpPut(urlString + "/" + deposition_id);
-		post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
+		post.setHeader("Authorization", "Bearer " + access_token);
 		post.addHeader("Content-Type", "application/json");
 		StringEntity entity = new StringEntity(metadata, StandardCharsets.UTF_8);
 		post.setEntity(entity);
 
 		HttpResponse response = client.execute(post);
-		System.out.println(response.getStatusLine().getStatusCode());
+		return response.getStatusLine().getStatusCode();
 
 
 	}
 
-	public void publish() throws IOException {
+	public int publish() throws IOException {
 		HttpClient client = new DefaultHttpClient();
 		HttpPost post = new HttpPost(urlString +"/"+ deposition_id +"/actions/publish") ;
-		post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
+		post.setHeader("Authorization", "Bearer " + access_token);
 
 		HttpResponse response = client.execute(post);
 
-		System.out.println(response.getStatusLine().getStatusCode());
+		return response.getStatusLine().getStatusCode();
 	}
 
 }

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java

@@ -0,0 +1,89 @@
+package eu.dnetlib.dhp.oa.graph.dump;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.DbClient;
+import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator;
+import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata;
+import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel;
+import eu.dnetlib.dhp.schema.common.ModelSupport;
+import eu.dnetlib.dhp.schema.oaf.Relation;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.Serializable;
+import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+public class SendToZenodo implements Serializable {
+
+    private static final Log log = LogFactory.getLog(SendToZenodo.class);
+
+
+    public static void main(final String[] args) throws Exception {
+        final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+                IOUtils
+                        .toString(
+                                SendToZenodo.class
+                                        .getResourceAsStream(
+                                                "/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json")));
+
+        parser.parseArgument(args);
+
+
+        final String hdfsPath = parser.get("hdfsPath");
+        final String hdfsNameNode = parser.get("hdfsNameNode");
+        final String access_token = parser.get("accessToken");
+        final String connection_url = parser.get("url");
+        final String metadata = parser.get("metadata");
+        Configuration conf = new Configuration();
+        conf.set("fs.defaultFS", hdfsNameNode);
+
+        FileSystem fileSystem = FileSystem.get(conf);
+
+
+        RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem.listFiles(
+               new Path(hdfsPath), true);
+        APIClient apiClient = new APIClient(connection_url, access_token);
+        apiClient.connect();
+        while(fileStatusListIterator.hasNext()){
+            LocatedFileStatus fileStatus = fileStatusListIterator.next();
+
+            Path p = fileStatus.getPath();
+            String p_string = p.toString();
+            String tmp = p_string.substring(0, p_string.lastIndexOf("/") );
+            String community = tmp.substring(tmp.lastIndexOf("/") + 1);
+            log.info("Sending information for community: " + community);
+            fileSystem.copyToLocalFile(p, new Path("/tmp/" + community));
+
+
+            File f = new File("/tmp/" + community);
+            apiClient.upload(f, community);
+            apiClient.sendMretadata(metadata);
+            apiClient.publish();
+
+            if (f.exists()){
+                f.delete();
+            }
+        }
+
+
+
+
+
+    }
+
+
+}