From 87856467e2a242dfaa7697357682d45c5a62f0f8 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 10 Aug 2020 11:38:41 +0200 Subject: [PATCH] removed isLookUpUrl and added code to read from HDSF the communitymap --- .../dhp/oa/graph/dump/SendToZenodoHDFS.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java index 40afc2538..0154dad99 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java @@ -1,18 +1,23 @@ package eu.dnetlib.dhp.oa.graph.dump; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.io.Serializable; import java.util.Optional; -import eu.dnetlib.dhp.common.api.ZenodoAPIClient; -import eu.dnetlib.dhp.common.api.MissingConceptDoiException; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; +import com.google.gson.Gson; + import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.api.MissingConceptDoiException; +import eu.dnetlib.dhp.common.api.ZenodoAPIClient; import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap; import eu.dnetlib.dhp.utils.ISLookupClientFactory; @@ -35,28 +40,27 @@ public class SendToZenodoHDFS implements Serializable { final String access_token = parser.get("accessToken"); final String connection_url = parser.get("connectionUrl"); final String metadata = parser.get("metadata"); - final String isLookUpUrl = parser.get("isLookUpUrl"); final Boolean newDeposition = Boolean.valueOf(parser.get("newDeposition")); - final String concept_rec_id = Optional.ofNullable(parser.get("conceptRecordId")) - .orElse(null); - - QueryInformationSystem qis = new QueryInformationSystem(); - qis.setIsLookUp(ISLookupClientFactory.getLookUpService(isLookUpUrl)); - CommunityMap communityMap = qis.getCommunityMap(); + final String concept_rec_id = Optional + .ofNullable(parser.get("conceptRecordId")) + .orElse(null); + final String communityMapPath = parser.get("communityMapPath"); Configuration conf = new Configuration(); conf.set("fs.defaultFS", hdfsNameNode); FileSystem fileSystem = FileSystem.get(conf); + CommunityMap communityMap = readCommunityMap(fileSystem, communityMapPath); + RemoteIterator fileStatusListIterator = fileSystem .listFiles( new Path(hdfsPath), true); ZenodoAPIClient zenodoApiClient = new ZenodoAPIClient(connection_url, access_token); - if (newDeposition){ + if (newDeposition) { zenodoApiClient.newDeposition(); - }else{ - if (concept_rec_id == null){ + } else { + if (concept_rec_id == null) { throw new MissingConceptDoiException("No concept record id has been provided"); } zenodoApiClient.newVersion(concept_rec_id); @@ -85,4 +89,20 @@ public class SendToZenodoHDFS implements Serializable { } + public static CommunityMap readCommunityMap(FileSystem fileSystem, String communityMapPath) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(fileSystem.open(new Path(communityMapPath)))); + StringBuffer sb = new StringBuffer(); + try { + String line; + while ((line = br.readLine()) != null) { + sb.append(line); + } + } finally { + br.close(); + + } + + return new Gson().fromJson(sb.toString(), CommunityMap.class); + } + }