forked from D-Net/dnet-hadoop
Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-hadoop
This commit is contained in:
commit
9cfc124ac5
|
@ -19,7 +19,7 @@
|
||||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
|
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
|
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
|
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement" value="common_lines"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
|
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="false"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
|
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
|
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
|
||||||
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
|
<setting id="org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration" value="common_lines"/>
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.common.api;
|
package eu.dnetlib.dhp.common.api;
|
||||||
|
|
||||||
public class MissingConceptDoiException extends Throwable {
|
public class MissingConceptDoiException extends Throwable {
|
||||||
public MissingConceptDoiException(String message) {
|
public MissingConceptDoiException(String message) {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,14 +12,12 @@ import okhttp3.*;
|
||||||
|
|
||||||
public class ZenodoAPIClient implements Serializable {
|
public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
|
|
||||||
String urlString;
|
String urlString;
|
||||||
String bucket;
|
String bucket;
|
||||||
|
|
||||||
String deposition_id;
|
String deposition_id;
|
||||||
String access_token;
|
String access_token;
|
||||||
|
|
||||||
|
|
||||||
public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
|
public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
|
||||||
|
|
||||||
private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
|
private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip");
|
||||||
|
@ -40,7 +38,9 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
this.bucket = bucket;
|
this.bucket = bucket;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setDeposition_id(String deposition_id){this.deposition_id = deposition_id;}
|
public void setDeposition_id(String deposition_id) {
|
||||||
|
this.deposition_id = deposition_id;
|
||||||
|
}
|
||||||
|
|
||||||
public ZenodoAPIClient(String urlString, String access_token) throws IOException {
|
public ZenodoAPIClient(String urlString, String access_token) throws IOException {
|
||||||
|
|
||||||
|
@ -182,10 +182,10 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
.post(RequestBody.create(MEDIA_TYPE_JSON, json))
|
.post(RequestBody.create(MEDIA_TYPE_JSON, json))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
|
@ -205,9 +205,9 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
|
ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
|
||||||
|
|
||||||
for(ZenodoModel zm : zenodoModelList){
|
for (ZenodoModel zm : zenodoModelList) {
|
||||||
if (zm.getConceptrecid().equals(concept_rec_id)){
|
if (zm.getConceptrecid().equals(concept_rec_id)) {
|
||||||
deposition_id = zm.getId();
|
deposition_id = zm.getId();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -220,11 +220,11 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString)
|
.url(urlString)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader("Content-Type", "application/json") // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
|
@ -241,11 +241,11 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader("Content-Type", "application/json") // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
|
@ -255,12 +255,10 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
// Get response body
|
// Get response body
|
||||||
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
|
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
|
||||||
|
|
||||||
|
|
||||||
return zenodoModel.getLinks().getBucket();
|
return zenodoModel.getLinks().getBucket();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.common.api.zenodo;
|
package eu.dnetlib.dhp.common.api.zenodo;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
|
|
@ -1,35 +1,36 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.dump.oaf;
|
package eu.dnetlib.dhp.schema.dump.oaf;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
public class Funder implements Serializable {
|
public class Funder implements Serializable {
|
||||||
private String shortName;
|
private String shortName;
|
||||||
|
|
||||||
private String name;
|
private String name;
|
||||||
|
|
||||||
private String jurisdiction;
|
private String jurisdiction;
|
||||||
|
|
||||||
public String getJurisdiction() {
|
public String getJurisdiction() {
|
||||||
return jurisdiction;
|
return jurisdiction;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setJurisdiction(String jurisdiction) {
|
public void setJurisdiction(String jurisdiction) {
|
||||||
this.jurisdiction = jurisdiction;
|
this.jurisdiction = jurisdiction;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getShortName() {
|
public String getShortName() {
|
||||||
return shortName;
|
return shortName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setShortName(String shortName) {
|
public void setShortName(String shortName) {
|
||||||
this.shortName = shortName;
|
this.shortName = shortName;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getName() {
|
public String getName() {
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setName(String name) {
|
public void setName(String name) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,11 +30,11 @@ public class Instance implements Serializable {
|
||||||
|
|
||||||
private String type;
|
private String type;
|
||||||
|
|
||||||
private KeyValue hostedby;
|
private KeyValue hostedby;
|
||||||
|
|
||||||
private List<String> url;
|
private List<String> url;
|
||||||
|
|
||||||
private KeyValue collectedfrom;
|
private KeyValue collectedfrom;
|
||||||
|
|
||||||
private String publicationdate;// dateofacceptance;
|
private String publicationdate;// dateofacceptance;
|
||||||
|
|
||||||
|
|
|
@ -1,44 +1,45 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.dump.oaf;
|
package eu.dnetlib.dhp.schema.dump.oaf;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
public class Project implements Serializable {
|
public class Project implements Serializable {
|
||||||
protected String id;// OpenAIRE id
|
protected String id;// OpenAIRE id
|
||||||
protected String code;
|
protected String code;
|
||||||
|
|
||||||
protected String acronym;
|
protected String acronym;
|
||||||
|
|
||||||
protected String title;
|
protected String title;
|
||||||
|
|
||||||
public String getId() {
|
public String getId() {
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setId(String id) {
|
public void setId(String id) {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getCode() {
|
public String getCode() {
|
||||||
return code;
|
return code;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setCode(String code) {
|
public void setCode(String code) {
|
||||||
this.code = code;
|
this.code = code;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getAcronym() {
|
public String getAcronym() {
|
||||||
return acronym;
|
return acronym;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setAcronym(String acronym) {
|
public void setAcronym(String acronym) {
|
||||||
this.acronym = acronym;
|
this.acronym = acronym;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getTitle() {
|
public String getTitle() {
|
||||||
return title;
|
return title;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setTitle(String title) {
|
public void setTitle(String title) {
|
||||||
this.title = title;
|
this.title = title;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,6 @@ import eu.dnetlib.dhp.schema.dump.oaf.KeyValue;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.Qualifier;
|
import eu.dnetlib.dhp.schema.dump.oaf.Qualifier;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.community.Project;
|
import eu.dnetlib.dhp.schema.dump.oaf.community.Project;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* To represent the generic organizaiton. It has the following parameters:
|
* To represent the generic organizaiton. It has the following parameters:
|
||||||
* - private String legalshortname to store the legalshortname of the organizaiton
|
* - private String legalshortname to store the legalshortname of the organizaiton
|
||||||
|
|
|
@ -366,48 +366,47 @@ public class ResultMapper implements Serializable {
|
||||||
|
|
||||||
private static Instance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i, boolean graph) {
|
private static Instance getInstance(eu.dnetlib.dhp.schema.oaf.Instance i, boolean graph) {
|
||||||
|
|
||||||
Instance instance = new Instance();
|
Instance instance = new Instance();
|
||||||
|
|
||||||
if(!graph){
|
if (!graph) {
|
||||||
|
instance
|
||||||
|
.setCollectedfrom(
|
||||||
|
KeyValue
|
||||||
|
.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue()));
|
||||||
|
instance
|
||||||
|
.setHostedby(
|
||||||
|
KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
|
||||||
|
.ofNullable(i.getAccessright());
|
||||||
|
if (opAr.isPresent()) {
|
||||||
|
if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
|
||||||
|
String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
|
||||||
instance
|
instance
|
||||||
.setCollectedfrom(
|
.setAccessright(
|
||||||
KeyValue
|
AccessRight
|
||||||
.newInstance(i.getCollectedfrom().getKey(), i.getCollectedfrom().getValue()));
|
.newInstance(
|
||||||
instance
|
code,
|
||||||
.setHostedby(
|
Constants.coarCodeLabelMap.get(code),
|
||||||
KeyValue.newInstance(i.getHostedby().getKey(), i.getHostedby().getValue()));
|
Constants.COAR_ACCESS_RIGHT_SCHEMA));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Optional<eu.dnetlib.dhp.schema.oaf.Qualifier> opAr = Optional
|
Optional
|
||||||
.ofNullable(i.getAccessright());
|
.ofNullable(i.getLicense())
|
||||||
if (opAr.isPresent()) {
|
.ifPresent(value -> instance.setLicense(value.getValue()));
|
||||||
if (Constants.accessRightsCoarMap.containsKey(opAr.get().getClassid())) {
|
Optional
|
||||||
String code = Constants.accessRightsCoarMap.get(opAr.get().getClassid());
|
.ofNullable(i.getDateofacceptance())
|
||||||
instance
|
.ifPresent(value -> instance.setPublicationdate(value.getValue()));
|
||||||
.setAccessright(
|
Optional
|
||||||
AccessRight
|
.ofNullable(i.getRefereed())
|
||||||
.newInstance(
|
.ifPresent(value -> instance.setRefereed(value.getClassname()));
|
||||||
code,
|
// .ifPresent(value -> instance.setRefereed(value.getValue()));
|
||||||
Constants.coarCodeLabelMap.get(code),
|
Optional
|
||||||
Constants.COAR_ACCESS_RIGHT_SCHEMA));
|
.ofNullable(i.getInstancetype())
|
||||||
}
|
.ifPresent(value -> instance.setType(value.getClassname()));
|
||||||
}
|
Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
|
||||||
|
|
||||||
|
|
||||||
Optional
|
|
||||||
.ofNullable(i.getLicense())
|
|
||||||
.ifPresent(value -> instance.setLicense(value.getValue()));
|
|
||||||
Optional
|
|
||||||
.ofNullable(i.getDateofacceptance())
|
|
||||||
.ifPresent(value -> instance.setPublicationdate(value.getValue()));
|
|
||||||
Optional
|
|
||||||
.ofNullable(i.getRefereed())
|
|
||||||
.ifPresent(value -> instance.setRefereed(value.getClassname()));
|
|
||||||
// .ifPresent(value -> instance.setRefereed(value.getValue()));
|
|
||||||
Optional
|
|
||||||
.ofNullable(i.getInstancetype())
|
|
||||||
.ifPresent(value -> instance.setType(value.getClassname()));
|
|
||||||
Optional.ofNullable(i.getUrl()).ifPresent(value -> instance.setUrl(value));
|
|
||||||
|
|
||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,11 +3,13 @@ package eu.dnetlib.dhp.oa.graph.dump;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.*;
|
import org.apache.hadoop.fs.*;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
|
import eu.dnetlib.dhp.common.api.MissingConceptDoiException;
|
||||||
import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
|
import eu.dnetlib.dhp.common.api.ZenodoAPIClient;
|
||||||
|
|
|
@ -8,8 +8,6 @@ import java.io.StringReader;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.graph.Funder;
|
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.graph.Project;
|
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.sql.Encoders;
|
import org.apache.spark.sql.Encoders;
|
||||||
import org.apache.spark.sql.SaveMode;
|
import org.apache.spark.sql.SaveMode;
|
||||||
|
@ -24,6 +22,8 @@ import eu.dnetlib.dhp.oa.graph.dump.Utils;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.*;
|
import eu.dnetlib.dhp.schema.dump.oaf.*;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.graph.*;
|
import eu.dnetlib.dhp.schema.dump.oaf.graph.*;
|
||||||
|
import eu.dnetlib.dhp.schema.dump.oaf.graph.Funder;
|
||||||
|
import eu.dnetlib.dhp.schema.dump.oaf.graph.Project;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Field;
|
import eu.dnetlib.dhp.schema.oaf.Field;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Journal;
|
import eu.dnetlib.dhp.schema.oaf.Journal;
|
||||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
|
|
||||||
|
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.graph;
|
package eu.dnetlib.dhp.oa.graph.dump.graph;
|
||||||
|
|
||||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||||
|
|
|
@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.graph.dump;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.JavaRDD;
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
|
@ -16,7 +17,9 @@ import org.junit.jupiter.api.BeforeAll;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.community.CommunitySplit;
|
import eu.dnetlib.dhp.oa.graph.dump.community.CommunitySplit;
|
||||||
import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
|
import eu.dnetlib.dhp.schema.dump.oaf.community.CommunityResult;
|
||||||
|
|
||||||
|
|
|
@ -38,7 +38,6 @@ public class ExtractRelationFromEntityTest {
|
||||||
private static final Logger log = LoggerFactory
|
private static final Logger log = LoggerFactory
|
||||||
.getLogger(ExtractRelationFromEntityTest.class);
|
.getLogger(ExtractRelationFromEntityTest.class);
|
||||||
|
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
public static void beforeAll() throws IOException {
|
public static void beforeAll() throws IOException {
|
||||||
workingDir = Files
|
workingDir = Files
|
||||||
|
|
Loading…
Reference in New Issue