D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

-

This commit is contained in:
Miriam Baglioni 2020-06-18 18:38:54 +02:00
parent 57b14af231
commit 44a12d244f
2 changed files with 111 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/APIClient.java
View File

@ -5,12 +5,19 @@ import java.io.*;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
@ -27,7 +34,7 @@ public class APIClient implements Serializable {
String bucket;
String deposition_id;
final String ACCESS_TOKEN = "5ImUj0VC1ICg4ifK5dc3AGzJhcfAB4osxrFlsr8WxHXxjaYgCE0hY8HZcDoe";
String access_token;
public String getUrlString() {
return urlString;
@ -45,10 +52,10 @@ public class APIClient implements Serializable {
this.bucket = bucket;
}
public APIClient(String urlString) throws IOException {
public APIClient(String urlString, String access_token) throws IOException {
this.urlString = urlString;
// connect();
this.access_token = access_token;
}
public int connect() throws IOException {
@ -62,7 +69,7 @@ public class APIClient implements Serializable {
StringEntity input = new StringEntity(json);
post.setEntity(input);
post.addHeader("Content-Type", "application/json");
post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
post.setHeader("Authorization", "Bearer " + access_token);
HttpResponse response = client.execute(post);
@ -77,48 +84,45 @@ public class APIClient implements Serializable {
}
public void upload(String filePath, String file_name) throws IOException {
public int upload(File file, String file_name) throws IOException {
HttpClient client = new DefaultHttpClient();
File file = new File(filePath);
HttpPut put = new HttpPut(bucket + "/" + file_name);
put.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
put.setHeader("Authorization", "Bearer " + access_token);
put.addHeader("Content-Type", "application/zip");
HttpEntity data = MultipartEntityBuilder.create().addBinaryBody(file_name, file).build();
put.setEntity(data);
HttpResponse response = client.execute(put);
String json = EntityUtils.toString(response.getEntity());
ZenodoModel newSubmission = new Gson().fromJson(json, ZenodoModel.class);
System.out.println(response.getStatusLine().getStatusCode());
return response.getStatusLine().getStatusCode();
}
public void sendMretadata(String metadata) throws IOException {
public int sendMretadata(String metadata) throws IOException {
HttpClient client = new DefaultHttpClient();
HttpPut post = new HttpPut(urlString + "/" + deposition_id);
post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
post.setHeader("Authorization", "Bearer " + access_token);
post.addHeader("Content-Type", "application/json");
StringEntity entity = new StringEntity(metadata, StandardCharsets.UTF_8);
post.setEntity(entity);
HttpResponse response = client.execute(post);
System.out.println(response.getStatusLine().getStatusCode());
return response.getStatusLine().getStatusCode();
}
public void publish() throws IOException {
public int publish() throws IOException {
HttpClient client = new DefaultHttpClient();
HttpPost post = new HttpPost(urlString +"/"+ deposition_id +"/actions/publish") ;
post.setHeader("Authorization", "Bearer " + ACCESS_TOKEN);
post.setHeader("Authorization", "Bearer " + access_token);
HttpResponse response = client.execute(post);
System.out.println(response.getStatusLine().getStatusCode());
return response.getStatusLine().getStatusCode();
}
}

89
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodo.java Normal file
View File

@ -0,0 +1,89 @@
package eu.dnetlib.dhp.oa.graph.dump;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.DbClient;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.Creator;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.Metadata;
import eu.dnetlib.dhp.oa.graph.dump.zenodo.ZenodoModel;
import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.Relation;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import java.io.BufferedWriter;
import java.io.File;
import java.io.Serializable;
import java.sql.ResultSet;
import java.util.Arrays;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
public class SendToZenodo implements Serializable {
private static final Log log = LogFactory.getLog(SendToZenodo.class);
public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
IOUtils
.toString(
SendToZenodo.class
.getResourceAsStream(
"/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json")));
parser.parseArgument(args);
final String hdfsPath = parser.get("hdfsPath");
final String hdfsNameNode = parser.get("hdfsNameNode");
final String access_token = parser.get("accessToken");
final String connection_url = parser.get("url");
final String metadata = parser.get("metadata");
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf);
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem.listFiles(
new Path(hdfsPath), true);
APIClient apiClient = new APIClient(connection_url, access_token);
apiClient.connect();
while(fileStatusListIterator.hasNext()){
LocatedFileStatus fileStatus = fileStatusListIterator.next();
Path p = fileStatus.getPath();
String p_string = p.toString();
String tmp = p_string.substring(0, p_string.lastIndexOf("/") );
String community = tmp.substring(tmp.lastIndexOf("/") + 1);
log.info("Sending information for community: " + community);
fileSystem.copyToLocalFile(p, new Path("/tmp/" + community));
File f = new File("/tmp/" + community);
apiClient.upload(f, community);
apiClient.sendMretadata(metadata);
apiClient.publish();
if (f.exists()){
f.delete();
}
}
}
}