2020-08-05 14:17:40 +02:00
|
|
|
|
|
|
|
package eu.dnetlib.dhp.oa.graph.dump;
|
|
|
|
|
2020-08-10 11:38:41 +02:00
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.io.InputStreamReader;
|
2020-08-07 17:49:14 +02:00
|
|
|
import java.io.Serializable;
|
|
|
|
import java.util.Optional;
|
|
|
|
|
2020-08-05 14:17:40 +02:00
|
|
|
import org.apache.commons.io.IOUtils;
|
|
|
|
import org.apache.commons.logging.Log;
|
|
|
|
import org.apache.commons.logging.LogFactory;
|
|
|
|
import org.apache.hadoop.conf.Configuration;
|
|
|
|
import org.apache.hadoop.fs.*;
|
|
|
|
|
2020-08-10 11:38:41 +02:00
|
|
|
import com.google.gson.Gson;
|
|
|
|
|
2020-08-07 17:49:14 +02:00
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
2020-08-10 11:38:41 +02:00
|
|
|
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
|
|
|
|
import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
|
2020-08-07 17:49:14 +02:00
|
|
|
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
|
|
|
|
import eu.dnetlib.dhp.utils.ISLookupClientFactory;
|
2020-08-05 14:17:40 +02:00
|
|
|
|
|
|
|
public class SendToZenodoHDFS implements Serializable {
|
|
|
|
|
|
|
|
private static final Log log = LogFactory.getLog(SendToZenodoHDFS.class);
|
|
|
|
|
2020-08-07 17:49:14 +02:00
|
|
|
public static void main(final String[] args) throws Exception, MissingConceptDoiException {
|
2020-08-05 14:17:40 +02:00
|
|
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
|
|
|
IOUtils
|
|
|
|
.toString(
|
|
|
|
SendToZenodoHDFS.class
|
|
|
|
.getResourceAsStream(
|
|
|
|
"/eu/dnetlib/dhp/oa/graph/dump/upload_zenodo.json")));
|
|
|
|
|
|
|
|
parser.parseArgument(args);
|
|
|
|
|
|
|
|
final String hdfsPath = parser.get("hdfsPath");
|
|
|
|
final String hdfsNameNode = parser.get("hdfsNameNode");
|
|
|
|
final String access_token = parser.get("accessToken");
|
|
|
|
final String connection_url = parser.get("connectionUrl");
|
|
|
|
final String metadata = parser.get("metadata");
|
2020-08-07 17:49:14 +02:00
|
|
|
final Boolean newDeposition = Boolean.valueOf(parser.get("newDeposition"));
|
2020-08-10 11:38:41 +02:00
|
|
|
final String concept_rec_id = Optional
|
|
|
|
.ofNullable(parser.get("conceptRecordId"))
|
|
|
|
.orElse(null);
|
|
|
|
final String communityMapPath = parser.get("communityMapPath");
|
2020-08-05 14:17:40 +02:00
|
|
|
|
|
|
|
Configuration conf = new Configuration();
|
|
|
|
conf.set("fs.defaultFS", hdfsNameNode);
|
|
|
|
|
|
|
|
FileSystem fileSystem = FileSystem.get(conf);
|
|
|
|
|
2020-08-10 11:38:41 +02:00
|
|
|
CommunityMap communityMap = readCommunityMap(fileSystem, communityMapPath);
|
|
|
|
|
2020-08-05 14:17:40 +02:00
|
|
|
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
|
|
|
.listFiles(
|
|
|
|
new Path(hdfsPath), true);
|
2020-08-07 17:49:14 +02:00
|
|
|
ZenodoAPIClient zenodoApiClient = new ZenodoAPIClient(connection_url, access_token);
|
2020-08-10 11:38:41 +02:00
|
|
|
if (newDeposition) {
|
2020-08-07 17:49:14 +02:00
|
|
|
zenodoApiClient.newDeposition();
|
2020-08-10 11:38:41 +02:00
|
|
|
} else {
|
|
|
|
if (concept_rec_id == null) {
|
2020-08-07 17:49:14 +02:00
|
|
|
throw new MissingConceptDoiException("No concept record id has been provided");
|
|
|
|
}
|
|
|
|
zenodoApiClient.newVersion(concept_rec_id);
|
|
|
|
}
|
|
|
|
|
2020-08-05 14:17:40 +02:00
|
|
|
while (fileStatusListIterator.hasNext()) {
|
|
|
|
LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
|
|
|
|
|
|
|
Path p = fileStatus.getPath();
|
|
|
|
String p_string = p.toString();
|
|
|
|
if (!p_string.endsWith("_SUCCESS")) {
|
|
|
|
String tmp = p_string.substring(0, p_string.lastIndexOf("/"));
|
|
|
|
String community = tmp.substring(tmp.lastIndexOf("/") + 1);
|
|
|
|
log.info("Sending information for community: " + community);
|
|
|
|
String community_name = communityMap.get(community).replace(" ", "_") + ".json.gz";
|
|
|
|
|
2020-08-07 17:49:14 +02:00
|
|
|
FSDataInputStream inputStream = fileSystem.open(p);
|
|
|
|
zenodoApiClient.uploadIS(inputStream, community_name, fileStatus.getLen());
|
2020-08-05 14:17:40 +02:00
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2020-08-07 17:49:14 +02:00
|
|
|
zenodoApiClient.sendMretadata(metadata);
|
|
|
|
zenodoApiClient.publish();
|
2020-08-05 14:17:40 +02:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2020-08-10 11:38:41 +02:00
|
|
|
public static CommunityMap readCommunityMap(FileSystem fileSystem, String communityMapPath) throws IOException {
|
|
|
|
BufferedReader br = new BufferedReader(new InputStreamReader(fileSystem.open(new Path(communityMapPath))));
|
|
|
|
StringBuffer sb = new StringBuffer();
|
|
|
|
try {
|
|
|
|
String line;
|
|
|
|
while ((line = br.readLine()) != null) {
|
|
|
|
sb.append(line);
|
|
|
|
}
|
|
|
|
} finally {
|
|
|
|
br.close();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return new Gson().fromJson(sb.toString(), CommunityMap.class);
|
|
|
|
}
|
|
|
|
|
2020-08-05 14:17:40 +02:00
|
|
|
}
|