D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

removed isLookUpUrl and added code to read from HDSF the communitymap

This commit is contained in:
Miriam Baglioni 2020-08-10 11:38:41 +02:00
parent 1cf7043e26
commit 87856467e2
1 changed files with 32 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java
View File

@ -1,18 +1,23 @@
package eu.dnetlib.dhp.oa.graph.dump; package eu.dnetlib.dhp.oa.graph.dump;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Serializable; import java.io.Serializable;
import java.util.Optional; import java.util.Optional;
import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.*;
import com.google.gson.Gson;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.dhp.utils.ISLookupClientFactory;
@ -35,20 +40,19 @@ public class SendToZenodoHDFS implements Serializable {
final String access_token = parser.get("accessToken"); final String access_token = parser.get("accessToken");
final String connection_url = parser.get("connectionUrl"); final String connection_url = parser.get("connectionUrl");
final String metadata = parser.get("metadata"); final String metadata = parser.get("metadata");
final String isLookUpUrl = parser.get("isLookUpUrl");
final Boolean newDeposition = Boolean.valueOf(parser.get("newDeposition")); final Boolean newDeposition = Boolean.valueOf(parser.get("newDeposition"));
final String concept_rec_id = Optional.ofNullable(parser.get("conceptRecordId")) final String concept_rec_id = Optional
.ofNullable(parser.get("conceptRecordId"))
.orElse(null); .orElse(null);
final String communityMapPath = parser.get("communityMapPath");
QueryInformationSystem qis = new QueryInformationSystem();
qis.setIsLookUp(ISLookupClientFactory.getLookUpService(isLookUpUrl));
CommunityMap communityMap = qis.getCommunityMap();
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode); conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf); FileSystem fileSystem = FileSystem.get(conf);
CommunityMap communityMap = readCommunityMap(fileSystem, communityMapPath);
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
.listFiles( .listFiles(
new Path(hdfsPath), true); new Path(hdfsPath), true);
@ -85,4 +89,20 @@ public class SendToZenodoHDFS implements Serializable {
} }
public static CommunityMap readCommunityMap(FileSystem fileSystem, String communityMapPath) throws IOException {
BufferedReader br = new BufferedReader(new InputStreamReader(fileSystem.open(new Path(communityMapPath))));
StringBuffer sb = new StringBuffer();
try {
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
}
} finally {
br.close();
}
return new Gson().fromJson(sb.toString(), CommunityMap.class);
}
} }