D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop

This commit is contained in:
Michele Artini 2020-09-08 16:39:54 +02:00
parent a597a218ab 8a523474b7
commit 9cfc124ac5
14 changed files with 123 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-build/dhp-code-style/src/main/resources/eclipse/formatter_dnet.xml
View File

@ -19,7 +19,7 @@
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/> <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/> <setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/> <setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/> <setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>

7
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java
View File

@ -1,7 +1,8 @@
package eu.dnetlib.dhp.common.api; package eu.dnetlib.dhp.common.api;
public class MissingConceptDoiException extends Throwable { public class MissingConceptDoiException extends Throwable {
public MissingConceptDoiException(String message) { public MissingConceptDoiException(String message) {
super(message); super(message);
} }
} }

42
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java
View File

@ -12,14 +12,12 @@ import okhttp3.*;
public class ZenodoAPIClient implements Serializable { public class ZenodoAPIClient implements Serializable {
String urlString; String urlString;
String bucket; String bucket;
String deposition_id; String deposition_id;
String access_token; String access_token;
public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8"); public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip"); private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
@ -40,7 +38,9 @@ public class ZenodoAPIClient implements Serializable {
this.bucket = bucket; this.bucket = bucket;
} }
public void setDeposition_id(String deposition_id){this.deposition_id = deposition_id;} public void setDeposition_id(String deposition_id) {
this.deposition_id = deposition_id;
}
public ZenodoAPIClient(String urlString, String access_token) throws IOException { public ZenodoAPIClient(String urlString, String access_token) throws IOException {
@ -182,10 +182,10 @@ public class ZenodoAPIClient implements Serializable {
OkHttpClient httpClient = new OkHttpClient(); OkHttpClient httpClient = new OkHttpClient();
Request request = new Request.Builder() Request request = new Request.Builder()
.url(urlString + "/" + deposition_id + "/actions/newversion") .url(urlString + "/" + deposition_id + "/actions/newversion")
.addHeader("Authorization", "Bearer " + access_token) .addHeader("Authorization", "Bearer " + access_token)
.post(RequestBody.create(MEDIA_TYPE_JSON, json)) .post(RequestBody.create(MEDIA_TYPE_JSON, json))
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {
@ -205,9 +205,9 @@ public class ZenodoAPIClient implements Serializable {
ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class); ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
for(ZenodoModel zm : zenodoModelList){ for (ZenodoModel zm : zenodoModelList) {
if (zm.getConceptrecid().equals(concept_rec_id)){ if (zm.getConceptrecid().equals(concept_rec_id)) {
deposition_id = zm.getId(); deposition_id = zm.getId();
return; return;
} }
} }
@ -220,11 +220,11 @@ public class ZenodoAPIClient implements Serializable {
OkHttpClient httpClient = new OkHttpClient(); OkHttpClient httpClient = new OkHttpClient();
Request request = new Request.Builder() Request request = new Request.Builder()
.url(urlString) .url(urlString)
.addHeader("Content-Type", "application/json") // add request headers .addHeader("Content-Type", "application/json") // add request headers
.addHeader("Authorization", "Bearer " + access_token) .addHeader("Authorization", "Bearer " + access_token)
.get() .get()
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {
@ -241,11 +241,11 @@ public class ZenodoAPIClient implements Serializable {
OkHttpClient httpClient = new OkHttpClient(); OkHttpClient httpClient = new OkHttpClient();
Request request = new Request.Builder() Request request = new Request.Builder()
.url(url) .url(url)
.addHeader("Content-Type", "application/json") // add request headers .addHeader("Content-Type", "application/json") // add request headers
.addHeader("Authorization", "Bearer " + access_token) .addHeader("Authorization", "Bearer " + access_token)
.get() .get()
.build(); .build();
try (Response response = httpClient.newCall(request).execute()) { try (Response response = httpClient.newCall(request).execute()) {
@ -255,12 +255,10 @@ public class ZenodoAPIClient implements Serializable {
// Get response body // Get response body
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class); ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
return zenodoModel.getLinks().getBucket(); return zenodoModel.getLinks().getBucket();
} }
} }
} }

1
dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java
View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.common.api.zenodo; package eu.dnetlib.dhp.common.api.zenodo;
import java.util.ArrayList; import java.util.ArrayList;

43
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/dump/oaf/Funder.java
View File

@ -1,35 +1,36 @@
package eu.dnetlib.dhp.schema.dump.oaf; package eu.dnetlib.dhp.schema.dump.oaf;
import java.io.Serializable; import java.io.Serializable;
public class Funder implements Serializable { public class Funder implements Serializable {
private String shortName; private String shortName;
private String name; private String name;
private String jurisdiction; private String jurisdiction;
public String getJurisdiction() { public String getJurisdiction() {
return jurisdiction; return jurisdiction;
} }
public void setJurisdiction(String jurisdiction) { public void setJurisdiction(String jurisdiction) {
this.jurisdiction = jurisdiction; this.jurisdiction = jurisdiction;
} }
public String getShortName() { public String getShortName() {
return shortName; return shortName;
} }
public void setShortName(String shortName) { public void setShortName(String shortName) {
this.shortName = shortName; this.shortName = shortName;
} }
public String getName() { public String getName() {
return name; return name;
} }
public void setName(String name) { public void setName(String name) {
this.name = name; this.name = name;
} }
} }

4
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/dump/oaf/Instance.java
View File

@ -30,11 +30,11 @@ public class Instance implements Serializable {
private String type; private String type;
private KeyValue hostedby; private KeyValue hostedby;
private List<String> url; private List<String> url;
private KeyValue collectedfrom; private KeyValue collectedfrom;
private String publicationdate;// dateofacceptance; private String publicationdate;// dateofacceptance;

57
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/dump/oaf/Project.java
View File

@ -1,44 +1,45 @@
package eu.dnetlib.dhp.schema.dump.oaf; package eu.dnetlib.dhp.schema.dump.oaf;
import java.io.Serializable; import java.io.Serializable;
public class Project implements Serializable { public class Project implements Serializable {
protected String id;// OpenAIRE id protected String id;// OpenAIRE id
protected String code; protected String code;
protected String acronym; protected String acronym;
protected String title; protected String title;
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public String getCode() { public String getCode() {
return code; return code;
} }
public void setCode(String code) { public void setCode(String code) {
this.code = code; this.code = code;
} }
public String getAcronym() { public String getAcronym() {
return acronym; return acronym;
} }
public void setAcronym(String acronym) { public void setAcronym(String acronym) {
this.acronym = acronym; this.acronym = acronym;
} }
public String getTitle() { public String getTitle() {
return title; return title;
} }
public void setTitle(String title) { public void setTitle(String title) {
this.title = title; this.title = title;
} }
} }

1
dhp-schemas/src/main/java/eu/dnetlib/dhp/schema/dump/oaf/graph/Organization.java
View File

@ -10,7 +10,6 @@ import eu.dnetlib.dhp.schema.dump.oaf.KeyValue;
import eu.dnetlib.dhp.schema.dump.oaf.Qualifier; import eu.dnetlib.dhp.schema.dump.oaf.Qualifier;
import eu.dnetlib.dhp.schema.dump.oaf.community.Project; import eu.dnetlib.dhp.schema.dump.oaf.community.Project;
/** /**
* To represent the generic organizaiton. It has the following parameters: * To represent the generic organizaiton. It has the following parameters:
* - private String legalshortname to store the legalshortname of the organizaiton * - private String legalshortname to store the legalshortname of the organizaiton

75
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java
View File

@ -366,48 +366,47 @@ public class ResultMapper implements Serializable {
private static Instance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i, boolean graph) { private static Instance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i, boolean graph) {
Instance instance = new Instance(); Instance instance = new Instance();
if(!graph){ if (!graph) {
instance
.setCollectedfrom(
KeyValue
.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue()));
instance
.setHostedby(
KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue()));
}
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
.ofNullable(i.getAccessright());
if (opAr.isPresent()) {
if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
instance instance
.setCollectedfrom( .setAccessright(
KeyValue AccessRight
.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue())); .newInstance(
instance code,
.setHostedby( Constants.coarCodeLabelMap.get(code),
KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue())); Constants.COAR_ACCESS_RIGHT_SCHEMA));
} }
}
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional Optional
.ofNullable(i.getAccessright()); .ofNullable(i.getLicense())
if (opAr.isPresent()) { .ifPresent(value -> instance.setLicense(value.getValue()));
if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) { Optional
String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid()); .ofNullable(i.getDateofacceptance())
instance .ifPresent(value -> instance.setPublicationdate(value.getValue()));
.setAccessright( Optional
AccessRight .ofNullable(i.getRefereed())
.newInstance( .ifPresent(value -> instance.setRefereed(value.getClassname()));
code, // .ifPresent(value -> instance.setRefereed(value.getValue()));
Constants.coarCodeLabelMap.get(code), Optional
Constants.COAR_ACCESS_RIGHT_SCHEMA)); .ofNullable(i.getInstancetype())
} .ifPresent(value -> instance.setType(value.getClassname()));
} Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
Optional
.ofNullable(i.getLicense())
.ifPresent(value -> instance.setLicense(value.getValue()));
Optional
.ofNullable(i.getDateofacceptance())
.ifPresent(value -> instance.setPublicationdate(value.getValue()));
Optional
.ofNullable(i.getRefereed())
.ifPresent(value -> instance.setRefereed(value.getClassname()));
// .ifPresent(value -> instance.setRefereed(value.getValue()));
Optional
.ofNullable(i.getInstancetype())
.ifPresent(value -> instance.setType(value.getClassname()));
Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
return instance; return instance;
} }

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SendToZenodoHDFS.java
View File

@ -3,11 +3,13 @@ package eu.dnetlib.dhp.oa.graph.dump;
import java.io.Serializable; import java.io.Serializable;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.*;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.api.MissingConceptDoiException; import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
import eu.dnetlib.dhp.common.api.ZenodoAPIClient; import eu.dnetlib.dhp.common.api.ZenodoAPIClient;

4
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/DumpGraphEntities.java
View File

@ -8,8 +8,6 @@ import java.io.StringReader;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.dump.oaf.graph.Funder;
import eu.dnetlib.dhp.schema.dump.oaf.graph.Project;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SaveMode;
@ -24,6 +22,8 @@ import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.dump.oaf.*; import eu.dnetlib.dhp.schema.dump.oaf.*;
import eu.dnetlib.dhp.schema.dump.oaf.graph.*; import eu.dnetlib.dhp.schema.dump.oaf.graph.*;
import eu.dnetlib.dhp.schema.dump.oaf.graph.Funder;
import eu.dnetlib.dhp.schema.dump.oaf.graph.Project;
import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.dhp.schema.oaf.Field;
import eu.dnetlib.dhp.schema.oaf.Journal; import eu.dnetlib.dhp.schema.oaf.Journal;
import eu.dnetlib.dhp.schema.oaf.OafEntity; import eu.dnetlib.dhp.schema.oaf.OafEntity;

1
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/SparkOrganizationRelation.java
View File

@ -1,5 +1,4 @@
package eu.dnetlib.dhp.oa.graph.dump.graph; package eu.dnetlib.dhp.oa.graph.dump.graph;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;

3
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/SplitForCommunityTest.java
View File

@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.graph.dump;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaRDD;
@ -16,7 +17,9 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunitySplit; import eu.dnetlib.dhp.oa.graph.dump.community.CommunitySplit;
import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult; import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;

1
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/dump/graph/ExtractRelationFromEntityTest.java
View File

@ -38,7 +38,6 @@ public class ExtractRelationFromEntityTest {
private static final Logger log = LoggerFactory private static final Logger log = LoggerFactory
.getLogger(ExtractRelationFromEntityTest.class); .getLogger(ExtractRelationFromEntityTest.class);
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
workingDir = Files workingDir = Files