andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of https://code-repo.d4science.org/D-Net/dnet-hadoop

This commit is contained in:
Claudio Atzori 2020-04-17 13:07:52 +02:00
parent cb0952428e 01ea7721f3
commit c92bfeeaee
4 changed files with 27 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/provision/update/Datacite2Scholix.java
View File

@ -34,8 +34,13 @@ public class Datacite2Scholix {
ScholixResource resource = generateDataciteScholixResource(dJson); ScholixResource resource = generateDataciteScholixResource(dJson);
return relIds.stream().flatMap(s-> { return relIds.stream().flatMap(s-> {
final List<Scholix> result = generateScholix(resource, s.get("relatedIdentifier"), s.get("relatedIdentifierType"), s.get("relationType"), updated); try {
final List<Scholix> result = generateScholix(resource, ""+s.get("relatedIdentifier"), s.get("relatedIdentifierType"), s.get("relationType"), updated);
return result.stream(); return result.stream();
} catch (Throwable e)
{
return new ArrayList<Scholix>().stream();
}
}).collect(Collectors.toList()); }).collect(Collectors.toList());
} }
@ -48,6 +53,7 @@ public class Datacite2Scholix {
} }
private List<Scholix> generateScholix(ScholixResource source, final String pid, final String pidtype, final String relType, final String updated) { private List<Scholix> generateScholix(ScholixResource source, final String pid, final String pidtype, final String relType, final String updated) {
if ("doi".equalsIgnoreCase(pidtype)) { if ("doi".equalsIgnoreCase(pidtype)) {
ScholixResource target = new ScholixResource(); ScholixResource target = new ScholixResource();
target.setIdentifier(Collections.singletonList(new ScholixIdentifier(pid, pidtype))); target.setIdentifier(Collections.singletonList(new ScholixIdentifier(pid, pidtype)));

4
dhp-workflows/dhp-graph-provision-scholexplorer/src/main/resources/eu/dnetlib/dhp/sx/synch/oozie_app/config-default.xml
View File

@ -7,4 +7,8 @@
<name>oozie.action.sharelib.for.spark</name> <name>oozie.action.sharelib.for.spark</name>
<value>spark2</value> <value>spark2</value>
</property> </property>
<property>
<name>oozie.launcher.mapreduce.user.classpath.first</name>
<value>true</value>
</property>
</configuration> </configuration>

19
dhp-workflows/dhp-graph-provision-scholexplorer/src/test/java/eu/dnetlib/dhp/provision/DataciteClientTest.java
View File

@ -3,16 +3,12 @@ package eu.dnetlib.dhp.provision;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.provision.scholix.Scholix; import eu.dnetlib.dhp.provision.scholix.Scholix;
import eu.dnetlib.dhp.provision.scholix.ScholixResource; import eu.dnetlib.dhp.provision.scholix.ScholixResource;
import eu.dnetlib.dhp.provision.update.CrossrefClient; import eu.dnetlib.dhp.provision.update.*;
import eu.dnetlib.dhp.provision.update.Datacite2Scholix;
import eu.dnetlib.dhp.provision.update.DataciteClient;
import eu.dnetlib.dhp.provision.update.DataciteClientIterator;
import eu.dnetlib.scholexplorer.relation.RelationMapper; import eu.dnetlib.scholexplorer.relation.RelationMapper;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List; import java.util.List;
@ -27,6 +23,19 @@ public class DataciteClientTest {
System.out.println(new ObjectMapper().writeValueAsString(s)); System.out.println(new ObjectMapper().writeValueAsString(s));
} }
// public void testS() throws Exception {
// RetrieveUpdateFromDatacite.main(new String[]{
// "-n", "file:///data/new_s2.txt",
// "-t", "/data/new_s2.txt",
// "-ts", "1586974078",
// "-ih", "ip-90-147-167-25.ct1.garrservices.it",
// "-in", "datacite",
// });
//
// }
public void testResolveDataset() throws Exception { public void testResolveDataset() throws Exception {
DataciteClient dc = new DataciteClient("ip-90-147-167-25.ct1.garrservices.it"); DataciteClient dc = new DataciteClient("ip-90-147-167-25.ct1.garrservices.it");
ScholixResource datasetByDOI = dc.getDatasetByDOI("10.17182/hepdata.15392.v1/t5"); ScholixResource datasetByDOI = dc.getDatasetByDOI("10.17182/hepdata.15392.v1/t5");