Merge pull request 'eoscAddOrganizationProjectsAndRelations' (#7) from eoscAddOrganizationProjectsAndRelations into eoscDump
Reviewed-on: #7
This commit is contained in:
commit
09b03c1c8a
|
@ -0,0 +1,49 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<parent>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-graph-dump</artifactId>
|
||||||
|
<version>1.2.5-SNAPSHOT</version>
|
||||||
|
</parent>
|
||||||
|
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>api</artifactId>
|
||||||
|
<version>1.2.5-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>dom4j</groupId>
|
||||||
|
<artifactId>dom4j</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>jaxen</groupId>
|
||||||
|
<artifactId>jaxen</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>dhp-common</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
|
<artifactId>jackson-annotations</artifactId>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
</project>
|
|
@ -0,0 +1,75 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 06/10/23
|
||||||
|
*/
|
||||||
|
public class QueryCommunityAPI {
|
||||||
|
private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/";
|
||||||
|
|
||||||
|
private static String get(String geturl) throws IOException {
|
||||||
|
URL url = new URL(geturl);
|
||||||
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||||
|
conn.setDoOutput(true);
|
||||||
|
conn.setRequestMethod("GET");
|
||||||
|
|
||||||
|
int responseCode = conn.getResponseCode();
|
||||||
|
String body = getBody(conn);
|
||||||
|
conn.disconnect();
|
||||||
|
if (responseCode != HttpURLConnection.HTTP_OK)
|
||||||
|
throw new IOException("Unexpected code " + responseCode + body);
|
||||||
|
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String communities() throws IOException {
|
||||||
|
|
||||||
|
return get(PRODUCTION_BASE_URL + "community/communities");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String community(String id) throws IOException {
|
||||||
|
|
||||||
|
return get(PRODUCTION_BASE_URL + "community/" + id);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String communityDatasource(String id) throws IOException {
|
||||||
|
|
||||||
|
return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String communityPropagationOrganization(String id) throws IOException {
|
||||||
|
|
||||||
|
return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String communityProjects(String id, String page, String size) throws IOException {
|
||||||
|
|
||||||
|
return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getBody(HttpURLConnection conn) throws IOException {
|
||||||
|
String body = "{}";
|
||||||
|
try (BufferedReader br = new BufferedReader(
|
||||||
|
new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||||
|
StringBuilder response = new StringBuilder();
|
||||||
|
String responseLine = null;
|
||||||
|
while ((responseLine = br.readLine()) != null) {
|
||||||
|
response.append(responseLine.trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
body = response.toString();
|
||||||
|
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,30 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonAutoDetect;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
@JsonAutoDetect
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public class CommunityContentprovider {
|
||||||
|
private String openaireId;
|
||||||
|
|
||||||
|
private String enabled;
|
||||||
|
|
||||||
|
public String getEnabled() {
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEnabled(String enabled) {
|
||||||
|
this.enabled = enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOpenaireId() {
|
||||||
|
return openaireId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setOpenaireId(final String openaireId) {
|
||||||
|
this.openaireId = openaireId;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class CommunityEntityMap extends HashMap<String, List<String>> {
|
||||||
|
|
||||||
|
public CommunityEntityMap() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> get(String key) {
|
||||||
|
|
||||||
|
if (super.get(key) == null) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
return super.get(key);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,82 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 06/10/23
|
||||||
|
*/
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public class CommunityModel implements Serializable {
|
||||||
|
private String id;
|
||||||
|
private String name;
|
||||||
|
private String description;
|
||||||
|
|
||||||
|
private String status;
|
||||||
|
|
||||||
|
private String type;
|
||||||
|
|
||||||
|
private List<String> subjects;
|
||||||
|
|
||||||
|
private String zenodoCommunity;
|
||||||
|
|
||||||
|
public List<String> getSubjects() {
|
||||||
|
return subjects;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSubjects(List<String> subjects) {
|
||||||
|
this.subjects = subjects;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getZenodoCommunity() {
|
||||||
|
return zenodoCommunity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setZenodoCommunity(String zenodoCommunity) {
|
||||||
|
this.zenodoCommunity = zenodoCommunity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getType() {
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setType(String type) {
|
||||||
|
this.type = type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getStatus() {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStatus(String status) {
|
||||||
|
this.status = status;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDescription() {
|
||||||
|
return description;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDescription(String description) {
|
||||||
|
this.description = description;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 06/10/23
|
||||||
|
*/
|
||||||
|
public class CommunitySummary extends ArrayList<CommunityModel> implements Serializable {
|
||||||
|
public CommunitySummary() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,51 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 09/10/23
|
||||||
|
*/
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public class ContentModel implements Serializable {
|
||||||
|
private List<ProjectModel> content;
|
||||||
|
private Integer totalPages;
|
||||||
|
private Boolean last;
|
||||||
|
private Integer number;
|
||||||
|
|
||||||
|
public List<ProjectModel> getContent() {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setContent(List<ProjectModel> content) {
|
||||||
|
this.content = content;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getTotalPages() {
|
||||||
|
return totalPages;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTotalPages(Integer totalPages) {
|
||||||
|
this.totalPages = totalPages;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getLast() {
|
||||||
|
return last;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLast(Boolean last) {
|
||||||
|
this.last = last;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getNumber() {
|
||||||
|
return number;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setNumber(Integer number) {
|
||||||
|
this.number = number;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
public class DatasourceList extends ArrayList<CommunityContentprovider> implements Serializable {
|
||||||
|
public DatasourceList() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,16 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 09/10/23
|
||||||
|
*/
|
||||||
|
public class OrganizationList extends ArrayList<String> implements Serializable {
|
||||||
|
|
||||||
|
public OrganizationList() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,44 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.communityapi.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 09/10/23
|
||||||
|
*/
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public class ProjectModel implements Serializable {
|
||||||
|
|
||||||
|
private String openaireId;
|
||||||
|
|
||||||
|
private String funder;
|
||||||
|
|
||||||
|
private String gratId;
|
||||||
|
|
||||||
|
public String getFunder() {
|
||||||
|
return funder;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFunder(String funder) {
|
||||||
|
this.funder = funder;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGratId() {
|
||||||
|
return gratId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGratId(String gratId) {
|
||||||
|
this.gratId = gratId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOpenaireId() {
|
||||||
|
return openaireId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setOpenaireId(String openaireId) {
|
||||||
|
this.openaireId = openaireId;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,46 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 13/09/22
|
||||||
|
*/
|
||||||
|
public class Affiliation implements Serializable {
|
||||||
|
@JsonSchema(description = "the OpenAIRE id of the organizaiton")
|
||||||
|
private String id;
|
||||||
|
|
||||||
|
@JsonSchema(description = "the name of the organization")
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
@JsonSchema(description = "the list of pids we have in OpenAIRE for the organization")
|
||||||
|
private List<OrganizationPid> pid;
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<OrganizationPid> getPid() {
|
||||||
|
return pid;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPid(List<OrganizationPid> pid) {
|
||||||
|
this.pid = pid;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,58 +1,31 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.eosc.model;
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
import java.io.Serializable;
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author miriam.baglioni
|
* To store information about the funder funding the project related to the result. It extends
|
||||||
* @Date 26/01/23
|
* eu.dnetlib.dhp.schema.dump.oaf.Funder with the following parameter: - - private
|
||||||
|
* eu.dnetdlib.dhp.schema.dump.oaf.graph.Fundings funding_stream to store the fundingstream
|
||||||
*/
|
*/
|
||||||
public class Funder implements Serializable {
|
public class Funder extends FunderShort {
|
||||||
|
|
||||||
@JsonSchema(description = "The short name of the funder (EC)")
|
@JsonSchema(description = "Description of the funding stream")
|
||||||
private String shortName;
|
private Fundings funding_stream;
|
||||||
|
|
||||||
@JsonSchema(description = "The name of the funder (European Commission)")
|
public Fundings getFunding_stream() {
|
||||||
private String name;
|
return funding_stream;
|
||||||
|
|
||||||
@JsonSchema(
|
|
||||||
description = "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)")
|
|
||||||
private String jurisdiction;
|
|
||||||
|
|
||||||
public String getJurisdiction() {
|
|
||||||
return jurisdiction;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setJurisdiction(String jurisdiction) {
|
public void setFunding_stream(Fundings funding_stream) {
|
||||||
this.jurisdiction = jurisdiction;
|
this.funding_stream = funding_stream;
|
||||||
}
|
|
||||||
|
|
||||||
public String getShortName() {
|
|
||||||
return shortName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setShortName(String shortName) {
|
|
||||||
this.shortName = shortName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getName() {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setName(String name) {
|
|
||||||
this.name = name;
|
|
||||||
}
|
|
||||||
|
|
||||||
@JsonSchema(description = "Stream of funding (e.g. for European Commission can be H2020 or FP7)")
|
|
||||||
private String fundingStream;
|
|
||||||
|
|
||||||
public String getFundingStream() {
|
|
||||||
return fundingStream;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFundingStream(String fundingStream) {
|
|
||||||
this.fundingStream = fundingStream;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 26/01/23
|
||||||
|
*/
|
||||||
|
public class FunderShort implements Serializable {
|
||||||
|
|
||||||
|
@JsonSchema(description = "The short name of the funder (EC)")
|
||||||
|
private String shortName;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The name of the funder (European Commission)")
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
@JsonSchema(
|
||||||
|
description = "Geographical jurisdiction (e.g. for European Commission is EU, for Croatian Science Foundation is HR)")
|
||||||
|
private String jurisdiction;
|
||||||
|
|
||||||
|
public String getJurisdiction() {
|
||||||
|
return jurisdiction;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setJurisdiction(String jurisdiction) {
|
||||||
|
this.jurisdiction = jurisdiction;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getShortName() {
|
||||||
|
return shortName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setShortName(String shortName) {
|
||||||
|
this.shortName = shortName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonSchema(description = "Stream of funding (e.g. for European Commission can be H2020 or FP7)")
|
||||||
|
private String fundingStream;
|
||||||
|
|
||||||
|
public String getFundingStream() {
|
||||||
|
return fundingStream;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFundingStream(String fundingStream) {
|
||||||
|
this.fundingStream = fundingStream;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,46 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To store inforamtion about the funding stream. It has two parameters: - private String id to store the id of the
|
||||||
|
* fundings stream. The id is created by appending the shortname of the funder to the name of each level in the xml
|
||||||
|
* representing the fundng stream. For example: if the funder is the European Commission, the funding level 0 name is
|
||||||
|
* FP7, the funding level 1 name is SP3 and the funding level 2 name is PEOPLE then the id will be: EC::FP7::SP3::PEOPLE
|
||||||
|
* - private String description to describe the funding stream. It is created by concatenating the description of each
|
||||||
|
* funding level so for the example above the description would be: SEVENTH FRAMEWORK PROGRAMME - SP3-People -
|
||||||
|
* Marie-Curie Actions
|
||||||
|
*/
|
||||||
|
public class Fundings implements Serializable {
|
||||||
|
|
||||||
|
@JsonSchema(description = "Id of the funding stream")
|
||||||
|
private String id;
|
||||||
|
private String description;
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDescription() {
|
||||||
|
return description;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDescription(String description) {
|
||||||
|
this.description = description;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,69 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To describe the funded amount. It has the following parameters: - private String currency to store the currency of
|
||||||
|
* the fund - private float totalcost to store the total cost of the project - private float fundedamount to store the
|
||||||
|
* funded amount by the funder
|
||||||
|
*/
|
||||||
|
public class Granted implements Serializable {
|
||||||
|
@JsonSchema(description = "The currency of the granted amount (e.g. EUR)")
|
||||||
|
private String currency;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The total cost of the project")
|
||||||
|
private float totalcost;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The funded amount")
|
||||||
|
private float fundedamount;
|
||||||
|
|
||||||
|
public String getCurrency() {
|
||||||
|
return currency;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCurrency(String currency) {
|
||||||
|
this.currency = currency;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getTotalcost() {
|
||||||
|
return totalcost;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTotalcost(float totalcost) {
|
||||||
|
this.totalcost = totalcost;
|
||||||
|
}
|
||||||
|
|
||||||
|
public float getFundedamount() {
|
||||||
|
return fundedamount;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFundedamount(float fundedamount) {
|
||||||
|
this.fundedamount = fundedamount;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Granted newInstance(String currency, float totalcost, float fundedamount) {
|
||||||
|
Granted granted = new Granted();
|
||||||
|
granted.currency = currency;
|
||||||
|
granted.totalcost = totalcost;
|
||||||
|
granted.fundedamount = fundedamount;
|
||||||
|
return granted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Granted newInstance(String currency, float fundedamount) {
|
||||||
|
Granted granted = new Granted();
|
||||||
|
granted.currency = currency;
|
||||||
|
granted.fundedamount = fundedamount;
|
||||||
|
return granted;
|
||||||
|
}
|
||||||
|
}
|
|
@ -7,19 +7,72 @@ import java.util.List;
|
||||||
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author miriam.baglioni
|
* To represent the generic organizaiton. It has the following parameters:
|
||||||
* @Date 13/09/22
|
* - private String legalshortname to store the legalshortname of the organizaiton
|
||||||
|
* - private String legalname to store the legal name of the organization
|
||||||
|
* - private String websiteurl to store the websiteurl of the organization
|
||||||
|
* - private List<String> alternativenames to store the alternative names of the organization
|
||||||
|
* - private Country country to store the country of the organization
|
||||||
|
* - private String id to store the openaire id of the organization
|
||||||
|
* - private List<OrganizationPid> pid to store the list of pids for the organization
|
||||||
*/
|
*/
|
||||||
public class Organization implements Serializable {
|
public class Organization implements Serializable {
|
||||||
@JsonSchema(description = "the OpenAIRE id of the organizaiton")
|
private String legalshortname;
|
||||||
|
private String legalname;
|
||||||
|
private String websiteurl;
|
||||||
|
|
||||||
|
@JsonSchema(description = "Alternative names that identify the organisation")
|
||||||
|
private List<String> alternativenames;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The organisation country")
|
||||||
|
private Country country;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The OpenAIRE id for the organisation")
|
||||||
private String id;
|
private String id;
|
||||||
|
|
||||||
@JsonSchema(description = "the name of the organization")
|
@JsonSchema(description = "Persistent identifiers for the organisation i.e. isni 0000000090326370")
|
||||||
private String name;
|
|
||||||
|
|
||||||
@JsonSchema(description = "the list of pids we have in OpenAIRE for the organization")
|
|
||||||
private List<OrganizationPid> pid;
|
private List<OrganizationPid> pid;
|
||||||
|
|
||||||
|
public String getLegalshortname() {
|
||||||
|
return legalshortname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLegalshortname(String legalshortname) {
|
||||||
|
this.legalshortname = legalshortname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLegalname() {
|
||||||
|
return legalname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLegalname(String legalname) {
|
||||||
|
this.legalname = legalname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getWebsiteurl() {
|
||||||
|
return websiteurl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWebsiteurl(String websiteurl) {
|
||||||
|
this.websiteurl = websiteurl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getAlternativenames() {
|
||||||
|
return alternativenames;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAlternativenames(List<String> alternativenames) {
|
||||||
|
this.alternativenames = alternativenames;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Country getCountry() {
|
||||||
|
return country;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCountry(Country country) {
|
||||||
|
this.country = country;
|
||||||
|
}
|
||||||
|
|
||||||
public String getId() {
|
public String getId() {
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
|
@ -28,14 +81,6 @@ public class Organization implements Serializable {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getName() {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setName(String name) {
|
|
||||||
this.name = name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<OrganizationPid> getPid() {
|
public List<OrganizationPid> getPid() {
|
||||||
return pid;
|
return pid;
|
||||||
}
|
}
|
||||||
|
@ -43,4 +88,5 @@ public class Organization implements Serializable {
|
||||||
public void setPid(List<OrganizationPid> pid) {
|
public void setPid(List<OrganizationPid> pid) {
|
||||||
this.pid = pid;
|
this.pid = pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,4 +32,12 @@ public class OrganizationPid implements Serializable {
|
||||||
public void setValue(String value) {
|
public void setValue(String value) {
|
||||||
this.value = value;
|
this.value = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static OrganizationPid newInstance(String type, String value) {
|
||||||
|
OrganizationPid op = new OrganizationPid();
|
||||||
|
op.type = type;
|
||||||
|
op.value = value;
|
||||||
|
|
||||||
|
return op;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,49 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To store information about the ec programme for the project. It has the following parameters: - private String code
|
||||||
|
* to store the code of the programme - private String description to store the description of the programme
|
||||||
|
*/
|
||||||
|
public class Programme implements Serializable {
|
||||||
|
@JsonSchema(description = "The code of the programme")
|
||||||
|
private String code;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The description of the programme")
|
||||||
|
private String description;
|
||||||
|
|
||||||
|
public String getCode() {
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCode(String code) {
|
||||||
|
this.code = code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDescription() {
|
||||||
|
return description;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDescription(String description) {
|
||||||
|
this.description = description;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Programme newInstance(String code, String description) {
|
||||||
|
Programme p = new Programme();
|
||||||
|
p.code = code;
|
||||||
|
p.description = description;
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,56 +1,80 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.eosc.model;
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author miriam.baglioni
|
* This is the class representing the Project in the model used for the dumps of the whole graph. At the moment the dump
|
||||||
* @Date 26/01/23
|
* of the Projects differs from the other dumps because we do not create relations between Funders (Organization) and
|
||||||
|
* Projects but we put the information about the Funder within the Project representation. We also removed the
|
||||||
|
* collected from element from the Project. No relation between the Project and the Datasource entity from which it is
|
||||||
|
* collected will be created. We will never create relations between Project and Datasource. In case some relation will
|
||||||
|
* be extracted from the Project they will refer the Funder and will be of type ( organization -> funds -> project,
|
||||||
|
* project -> isFundedBy -> organization) We also removed the duration parameter because the most of times it is set to
|
||||||
|
* 0. It has the following parameters:
|
||||||
|
* - private String id to store the id of the project (OpenAIRE id)
|
||||||
|
* - private String websiteurl to store the websiteurl of the project
|
||||||
|
* - private String code to store the grant agreement of the project
|
||||||
|
* - private String acronym to store the acronym of the project
|
||||||
|
* - private String title to store the tile of the project
|
||||||
|
* - private String startdate to store the start date
|
||||||
|
* - private String enddate to store the end date
|
||||||
|
* - private String callidentifier to store the call indentifier
|
||||||
|
* - private String keywords to store the keywords
|
||||||
|
* - private boolean openaccessmandateforpublications to store if the project must accomplish to the open access mandate
|
||||||
|
* for publications. This value will be set to true if one of the field in the project represented in the internal model
|
||||||
|
* is set to true
|
||||||
|
* - private boolean openaccessmandatefordataset to store if the project must accomplish to the open access mandate for
|
||||||
|
* dataset. It is set to the value in the corresponding filed of the project represented in the internal model
|
||||||
|
* - private List<String> subject to store the list of subjects of the project
|
||||||
|
* - private List<Funder> funding to store the list of funder of the project
|
||||||
|
* - private String summary to store the summary of the project
|
||||||
|
* - private Granted granted to store the granted amount
|
||||||
|
* - private List<Programme> h2020programme to store the list of programmes the project is related to
|
||||||
*/
|
*/
|
||||||
|
|
||||||
public class Project implements Serializable {
|
public class Project implements Serializable {
|
||||||
@JsonSchema(description = "The OpenAIRE id for the project")
|
private String id;
|
||||||
protected String id;// OpenAIRE id
|
|
||||||
|
|
||||||
@JsonSchema(description = "The grant agreement number")
|
private String websiteurl;
|
||||||
protected String code;
|
private String code;
|
||||||
|
private String acronym;
|
||||||
|
private String title;
|
||||||
|
|
||||||
@JsonSchema(description = "The acronym of the project")
|
private String startdate;
|
||||||
protected String acronym;
|
|
||||||
|
|
||||||
protected String title;
|
private String enddate;
|
||||||
|
|
||||||
@JsonSchema(description = "Information about the funder funding the project")
|
private String callidentifier;
|
||||||
private Funder funder;
|
|
||||||
|
|
||||||
private Provenance provenance;
|
private String keywords;
|
||||||
|
|
||||||
private Validated validated;
|
private boolean openaccessmandateforpublications;
|
||||||
|
|
||||||
public void setValidated(Validated validated) {
|
private boolean openaccessmandatefordataset;
|
||||||
this.validated = validated;
|
private List<String> subject;
|
||||||
}
|
|
||||||
|
|
||||||
public Validated getValidated() {
|
@JsonSchema(description = "Funding information for the project")
|
||||||
return validated;
|
private List<Funder> funding;
|
||||||
}
|
|
||||||
|
|
||||||
public Provenance getProvenance() {
|
private String summary;
|
||||||
return provenance;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setProvenance(Provenance provenance) {
|
@JsonSchema(description = "The money granted to the project")
|
||||||
this.provenance = provenance;
|
private Granted granted;
|
||||||
}
|
|
||||||
|
|
||||||
public Funder getFunder() {
|
@JsonSchema(description = "The h2020 programme funding the project")
|
||||||
return funder;
|
private List<Programme> h2020programme;
|
||||||
}
|
|
||||||
|
|
||||||
public void setFunder(Funder funders) {
|
|
||||||
this.funder = funders;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getId() {
|
public String getId() {
|
||||||
return id;
|
return id;
|
||||||
|
@ -60,6 +84,14 @@ public class Project implements Serializable {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getWebsiteurl() {
|
||||||
|
return websiteurl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWebsiteurl(String websiteurl) {
|
||||||
|
this.websiteurl = websiteurl;
|
||||||
|
}
|
||||||
|
|
||||||
public String getCode() {
|
public String getCode() {
|
||||||
return code;
|
return code;
|
||||||
}
|
}
|
||||||
|
@ -84,14 +116,91 @@ public class Project implements Serializable {
|
||||||
this.title = title;
|
this.title = title;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Project newInstance(String id, String code, String acronym, String title, Funder funder) {
|
public String getStartdate() {
|
||||||
Project project = new Project();
|
return startdate;
|
||||||
project.setAcronym(acronym);
|
|
||||||
project.setCode(code);
|
|
||||||
project.setFunder(funder);
|
|
||||||
project.setId(id);
|
|
||||||
project.setTitle(title);
|
|
||||||
return project;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setStartdate(String startdate) {
|
||||||
|
this.startdate = startdate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getEnddate() {
|
||||||
|
return enddate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEnddate(String enddate) {
|
||||||
|
this.enddate = enddate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCallidentifier() {
|
||||||
|
return callidentifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCallidentifier(String callidentifier) {
|
||||||
|
this.callidentifier = callidentifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getKeywords() {
|
||||||
|
return keywords;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setKeywords(String keywords) {
|
||||||
|
this.keywords = keywords;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isOpenaccessmandateforpublications() {
|
||||||
|
return openaccessmandateforpublications;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setOpenaccessmandateforpublications(boolean openaccessmandateforpublications) {
|
||||||
|
this.openaccessmandateforpublications = openaccessmandateforpublications;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isOpenaccessmandatefordataset() {
|
||||||
|
return openaccessmandatefordataset;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setOpenaccessmandatefordataset(boolean openaccessmandatefordataset) {
|
||||||
|
this.openaccessmandatefordataset = openaccessmandatefordataset;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getSubject() {
|
||||||
|
return subject;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSubject(List<String> subject) {
|
||||||
|
this.subject = subject;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Funder> getFunding() {
|
||||||
|
return funding;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFunding(List<Funder> funding) {
|
||||||
|
this.funding = funding;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSummary() {
|
||||||
|
return summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSummary(String summary) {
|
||||||
|
this.summary = summary;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Granted getGranted() {
|
||||||
|
return granted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGranted(Granted granted) {
|
||||||
|
this.granted = granted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Programme> getH2020programme() {
|
||||||
|
return h2020programme;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setH2020programme(List<Programme> h2020programme) {
|
||||||
|
this.h2020programme = h2020programme;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,97 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.eosc.model;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 26/01/23
|
||||||
|
*/
|
||||||
|
public class ProjectSummary implements Serializable {
|
||||||
|
@JsonSchema(description = "The OpenAIRE id for the project")
|
||||||
|
protected String id;// OpenAIRE id
|
||||||
|
|
||||||
|
@JsonSchema(description = "The grant agreement number")
|
||||||
|
protected String code;
|
||||||
|
|
||||||
|
@JsonSchema(description = "The acronym of the project")
|
||||||
|
protected String acronym;
|
||||||
|
|
||||||
|
protected String title;
|
||||||
|
|
||||||
|
@JsonSchema(description = "Information about the funder funding the project")
|
||||||
|
private FunderShort funder;
|
||||||
|
|
||||||
|
private Provenance provenance;
|
||||||
|
|
||||||
|
private Validated validated;
|
||||||
|
|
||||||
|
public void setValidated(Validated validated) {
|
||||||
|
this.validated = validated;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Validated getValidated() {
|
||||||
|
return validated;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Provenance getProvenance() {
|
||||||
|
return provenance;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProvenance(Provenance provenance) {
|
||||||
|
this.provenance = provenance;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FunderShort getFunder() {
|
||||||
|
return funder;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFunder(FunderShort funders) {
|
||||||
|
this.funder = funders;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCode() {
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCode(String code) {
|
||||||
|
this.code = code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAcronym() {
|
||||||
|
return acronym;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAcronym(String acronym) {
|
||||||
|
this.acronym = acronym;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTitle() {
|
||||||
|
return title;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTitle(String title) {
|
||||||
|
this.title = title;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ProjectSummary newInstance(String id, String code, String acronym, String title, FunderShort funder) {
|
||||||
|
ProjectSummary project = new ProjectSummary();
|
||||||
|
project.setAcronym(acronym);
|
||||||
|
project.setCode(code);
|
||||||
|
project.setFunder(funder);
|
||||||
|
project.setId(id);
|
||||||
|
project.setTitle(title);
|
||||||
|
return project;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -4,6 +4,7 @@ package eu.dnetlib.dhp.eosc.model;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -20,12 +21,15 @@ public class Relation implements Serializable {
|
||||||
private String target;
|
private String target;
|
||||||
|
|
||||||
@JsonSchema(description = "To represent the semantics of a relation between two entities")
|
@JsonSchema(description = "To represent the semantics of a relation between two entities")
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
private RelType reltype;
|
private RelType reltype;
|
||||||
|
|
||||||
@JsonSchema(description = "The reason why OpenAIRE holds the relation ")
|
@JsonSchema(description = "The reason why OpenAIRE holds the relation ")
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
private Provenance provenance;
|
private Provenance provenance;
|
||||||
|
|
||||||
@JsonSchema(description = "The result type of the target for this relation")
|
@JsonSchema(description = "The result type of the target for this relation")
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
private String targetType;
|
private String targetType;
|
||||||
|
|
||||||
public String getTargetType() {
|
public String getTargetType() {
|
||||||
|
@ -82,4 +86,12 @@ public class Relation implements Serializable {
|
||||||
relation.provenance = provenance;
|
relation.provenance = provenance;
|
||||||
return relation;
|
return relation;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Relation newInstance(String source, String target) {
|
||||||
|
Relation relation = new Relation();
|
||||||
|
relation.source = source;
|
||||||
|
relation.target = target;
|
||||||
|
|
||||||
|
return relation;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,13 +24,13 @@ public class Result implements Serializable {
|
||||||
private List<String> keywords;
|
private List<String> keywords;
|
||||||
|
|
||||||
@JsonSchema(description = "The list of organizations the result is affiliated to")
|
@JsonSchema(description = "The list of organizations the result is affiliated to")
|
||||||
private List<Organization> affiliation;
|
private List<Affiliation> affiliation;
|
||||||
|
|
||||||
@JsonSchema(description = "The indicators for this result")
|
@JsonSchema(description = "The indicators for this result")
|
||||||
private Indicator indicator;
|
private Indicator indicator;
|
||||||
|
|
||||||
@JsonSchema(description = "List of projects (i.e. grants) that (co-)funded the production ofn the research results")
|
@JsonSchema(description = "List of projects (i.e. grants) that (co-)funded the production ofn the research results")
|
||||||
private List<Project> projects;
|
private List<ProjectSummary> projects;
|
||||||
|
|
||||||
@JsonSchema(
|
@JsonSchema(
|
||||||
description = "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu")
|
description = "Reference to a relevant research infrastructure, initiative or community (RI/RC) among those collaborating with OpenAIRE. Please see https://connect.openaire.eu")
|
||||||
|
@ -409,11 +409,11 @@ public class Result implements Serializable {
|
||||||
this.collectedfrom = collectedfrom;
|
this.collectedfrom = collectedfrom;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Project> getProjects() {
|
public List<ProjectSummary> getProjects() {
|
||||||
return projects;
|
return projects;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setProjects(List<Project> projects) {
|
public void setProjects(List<ProjectSummary> projects) {
|
||||||
this.projects = projects;
|
this.projects = projects;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -465,11 +465,11 @@ public class Result implements Serializable {
|
||||||
this.subject = subject;
|
this.subject = subject;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Organization> getAffiliation() {
|
public List<Affiliation> getAffiliation() {
|
||||||
return affiliation;
|
return affiliation;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setAffiliation(List<Organization> affiliation) {
|
public void setAffiliation(List<Affiliation> affiliation) {
|
||||||
this.affiliation = affiliation;
|
this.affiliation = affiliation;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
12
dump/pom.xml
12
dump/pom.xml
|
@ -54,7 +54,17 @@
|
||||||
<artifactId>dump-schema</artifactId>
|
<artifactId>dump-schema</artifactId>
|
||||||
<version>1.2.5-SNAPSHOT</version>
|
<version>1.2.5-SNAPSHOT</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>api</artifactId>
|
||||||
|
<version>1.2.5-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
|
<artifactId>api</artifactId>
|
||||||
|
<version>1.2.5-SNAPSHOT</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,55 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.graph.dump;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.communityapi.model.*;
|
||||||
|
import eu.dnetlib.dhp.oa.graph.dump.eosc.CommunityMap;
|
||||||
|
|
||||||
|
public class UtilCommunityAPI {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(UtilCommunityAPI.class);
|
||||||
|
|
||||||
|
public CommunityMap getCommunityMap(boolean singleCommunity, String communityId)
|
||||||
|
throws IOException {
|
||||||
|
if (singleCommunity)
|
||||||
|
return getMap(Arrays.asList(getCommunity(communityId)));
|
||||||
|
return getMap(getValidCommunities());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private CommunityMap getMap(List<CommunityModel> communities) {
|
||||||
|
final CommunityMap map = new CommunityMap();
|
||||||
|
communities.forEach(c -> map.put(c.getId(), c.getName()));
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<CommunityModel> getValidCommunities() throws IOException {
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
return mapper
|
||||||
|
.readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.communities(), CommunitySummary.class)
|
||||||
|
.stream()
|
||||||
|
.filter(
|
||||||
|
community -> (community.getStatus().equals("all") || community.getStatus().equalsIgnoreCase("public"))
|
||||||
|
&&
|
||||||
|
(community.getType().equals("ri") || community.getType().equals("community")))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private CommunityModel getCommunity(String id) throws IOException {
|
||||||
|
ObjectMapper mapper = new ObjectMapper();
|
||||||
|
return mapper
|
||||||
|
.readValue(eu.dnetlib.dhp.communityapi.QueryCommunityAPI.community(id), CommunityModel.class);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -19,6 +19,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Affiliation;
|
||||||
import eu.dnetlib.dhp.eosc.model.OrganizationPid;
|
import eu.dnetlib.dhp.eosc.model.OrganizationPid;
|
||||||
import eu.dnetlib.dhp.eosc.model.Result;
|
import eu.dnetlib.dhp.eosc.model.Result;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
@ -91,7 +92,7 @@ public class ExtendEoscResultWithOrganization implements Serializable {
|
||||||
if (t2._2() != null) {
|
if (t2._2() != null) {
|
||||||
ResultOrganizations rOrg = new ResultOrganizations();
|
ResultOrganizations rOrg = new ResultOrganizations();
|
||||||
rOrg.setResultId(t2._1().getTarget());
|
rOrg.setResultId(t2._1().getTarget());
|
||||||
eu.dnetlib.dhp.eosc.model.Organization org = new eu.dnetlib.dhp.eosc.model.Organization();
|
Affiliation org = new Affiliation();
|
||||||
org.setId(t2._2().getId());
|
org.setId(t2._2().getId());
|
||||||
if (Optional.ofNullable(t2._2().getLegalname()).isPresent()) {
|
if (Optional.ofNullable(t2._2().getLegalname()).isPresent()) {
|
||||||
org.setName(t2._2().getLegalname().getValue());
|
org.setName(t2._2().getLegalname().getValue());
|
||||||
|
@ -135,7 +136,7 @@ public class ExtendEoscResultWithOrganization implements Serializable {
|
||||||
return first._1();
|
return first._1();
|
||||||
}
|
}
|
||||||
Result ret = first._1();
|
Result ret = first._1();
|
||||||
List<eu.dnetlib.dhp.eosc.model.Organization> affiliation = new ArrayList<>();
|
List<Affiliation> affiliation = new ArrayList<>();
|
||||||
Set<String> alreadyInsertedAffiliations = new HashSet<>();
|
Set<String> alreadyInsertedAffiliations = new HashSet<>();
|
||||||
affiliation.add(first._2().getAffiliation());
|
affiliation.add(first._2().getAffiliation());
|
||||||
alreadyInsertedAffiliations.add(first._2().getAffiliation().getId());
|
alreadyInsertedAffiliations.add(first._2().getAffiliation().getId());
|
||||||
|
|
|
@ -5,6 +5,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
|
@ -15,10 +16,14 @@ import org.apache.spark.sql.Dataset;
|
||||||
import org.apache.spark.sql.Encoders;
|
import org.apache.spark.sql.Encoders;
|
||||||
import org.apache.spark.sql.SaveMode;
|
import org.apache.spark.sql.SaveMode;
|
||||||
import org.apache.spark.sql.SparkSession;
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.spark_project.jetty.util.StringUtil;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Affiliation;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Country;
|
||||||
import eu.dnetlib.dhp.eosc.model.OrganizationPid;
|
import eu.dnetlib.dhp.eosc.model.OrganizationPid;
|
||||||
import eu.dnetlib.dhp.eosc.model.Result;
|
import eu.dnetlib.dhp.eosc.model.Result;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
@ -33,6 +38,8 @@ import scala.Tuple2;
|
||||||
public class ExtendEoscResultWithOrganizationStep2 implements Serializable {
|
public class ExtendEoscResultWithOrganizationStep2 implements Serializable {
|
||||||
private static final Logger log = LoggerFactory.getLogger(ExtendEoscResultWithOrganizationStep2.class);
|
private static final Logger log = LoggerFactory.getLogger(ExtendEoscResultWithOrganizationStep2.class);
|
||||||
|
|
||||||
|
private final static String UNKNOWN = "UNKNOWN";
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
String jsonConfiguration = IOUtils
|
String jsonConfiguration = IOUtils
|
||||||
.toString(
|
.toString(
|
||||||
|
@ -52,27 +59,97 @@ public class ExtendEoscResultWithOrganizationStep2 implements Serializable {
|
||||||
final String inputPath = parser.get("sourcePath");
|
final String inputPath = parser.get("sourcePath");
|
||||||
log.info("inputPath: {}", inputPath);
|
log.info("inputPath: {}", inputPath);
|
||||||
|
|
||||||
final String resultPath = parser.get("resultPath");
|
final String workingPath = parser.get("workingPath");
|
||||||
log.info("resultPath: {}", resultPath);
|
log.info("workingPath: {}", workingPath);
|
||||||
|
|
||||||
final String outputPath = parser.get("outputPath");
|
final String outputPath = parser.get("outputPath");
|
||||||
log.info("outputPath: {}", outputPath);
|
log.info("outputPath: {}", outputPath);
|
||||||
|
|
||||||
|
final String resultType = parser.get("resultType");
|
||||||
|
log.info("resultType: {}", resultType);
|
||||||
|
|
||||||
SparkConf conf = new SparkConf();
|
SparkConf conf = new SparkConf();
|
||||||
|
|
||||||
runWithSparkSession(
|
runWithSparkSession(
|
||||||
conf,
|
conf,
|
||||||
isSparkSessionManaged,
|
isSparkSessionManaged,
|
||||||
spark -> {
|
spark -> {
|
||||||
Utils.removeOutputDir(spark, outputPath);
|
Utils.removeOutputDir(spark, workingPath + resultType + "extendedaffiliation");
|
||||||
addOrganizations(spark, inputPath, outputPath, resultPath);
|
Utils.removeOutputDir(spark, workingPath + resultType + "organization");
|
||||||
|
Utils.removeOutputDir(spark, workingPath + resultType + "resultOrganization");
|
||||||
|
addOrganizations(spark, inputPath, workingPath, resultType);
|
||||||
|
dumpOrganizationAndRelations(spark, inputPath, workingPath, resultType);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void addOrganizations(SparkSession spark, String inputPath, String outputPath,
|
private static void dumpOrganizationAndRelations(SparkSession spark, String inputPath, String workingPath,
|
||||||
String resultPath) {
|
String resultType) {
|
||||||
|
Dataset<Relation> relation = Utils
|
||||||
|
.readPath(spark, inputPath + "/relation", Relation.class)
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Relation>) r -> !r.getDataInfo().getDeletedbyinference() &&
|
||||||
|
r.getSubRelType().equalsIgnoreCase(ModelConstants.AFFILIATION));
|
||||||
|
|
||||||
|
Dataset<Organization> organization = Utils
|
||||||
|
.readPath(spark, inputPath + "/organization", Organization.class)
|
||||||
|
.filter((FilterFunction<Organization>) o -> !o.getDataInfo().getDeletedbyinference());
|
||||||
|
|
||||||
|
Dataset<Result> result = Utils.readPath(spark, workingPath + resultType, Result.class);
|
||||||
|
|
||||||
|
// result -> organization takes the relation of type affiliation having the source in the results related to
|
||||||
|
// EOSC
|
||||||
|
Dataset<Relation> eoscRelation = result
|
||||||
|
.joinWith(relation, result.col("id").equalTo(relation.col("source")))
|
||||||
|
.map((MapFunction<Tuple2<Result, Relation>, Relation>) t2 -> t2._2(), Encoders.bean(Relation.class));
|
||||||
|
|
||||||
|
log.info("Number of affiliation relation for " + resultType + " = " + eoscRelation.count());
|
||||||
|
|
||||||
|
// from eoscRelation select the organization
|
||||||
|
Dataset<String> organizationIds = eoscRelation
|
||||||
|
.joinWith(organization, eoscRelation.col("target").equalTo(organization.col("id")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<Relation, Organization>, String>) t2 -> t2._2().getId(),
|
||||||
|
Encoders.STRING())
|
||||||
|
.distinct();
|
||||||
|
|
||||||
|
organizationIds
|
||||||
|
.joinWith(organization, organizationIds.col("value").equalTo(organization.col("id")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<String, Organization>, eu.dnetlib.dhp.eosc.model.Organization>) t2 -> mapOrganization(
|
||||||
|
t2._2()),
|
||||||
|
Encoders.bean(eu.dnetlib.dhp.eosc.model.Organization.class))
|
||||||
|
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(workingPath + resultType + "organization");
|
||||||
|
|
||||||
|
eoscRelation
|
||||||
|
.joinWith(organization, eoscRelation.col("target").equalTo(organization.col("id")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<Relation, Organization>, eu.dnetlib.dhp.eosc.model.Relation>) t2 -> {
|
||||||
|
if (isToBeRemovedOrg(t2._2()))
|
||||||
|
return new eu.dnetlib.dhp.eosc.model.Relation();
|
||||||
|
return eu.dnetlib.dhp.eosc.model.Relation
|
||||||
|
.newInstance(t2._1().getSource(), t2._1().getTarget());
|
||||||
|
|
||||||
|
},
|
||||||
|
Encoders.bean(eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.filter((FilterFunction<eu.dnetlib.dhp.eosc.model.Relation>) r -> StringUtil.isNotBlank(r.getSource()))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(workingPath + resultType + "resultOrganization");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void addOrganizations(SparkSession spark, String inputPath, String workingPath,
|
||||||
|
String resultType) {
|
||||||
|
|
||||||
Dataset<Result> results = Utils
|
Dataset<Result> results = Utils
|
||||||
.readPath(spark, resultPath, Result.class);
|
.readPath(spark, workingPath + resultType, Result.class);
|
||||||
|
|
||||||
Dataset<Relation> relations = Utils
|
Dataset<Relation> relations = Utils
|
||||||
.readPath(spark, inputPath + "/relation", Relation.class)
|
.readPath(spark, inputPath + "/relation", Relation.class)
|
||||||
|
@ -80,15 +157,19 @@ public class ExtendEoscResultWithOrganizationStep2 implements Serializable {
|
||||||
(FilterFunction<Relation>) r -> !r.getDataInfo().getDeletedbyinference() &&
|
(FilterFunction<Relation>) r -> !r.getDataInfo().getDeletedbyinference() &&
|
||||||
!r.getDataInfo().getInvisible() && r.getSubRelType().equalsIgnoreCase(ModelConstants.AFFILIATION));
|
!r.getDataInfo().getInvisible() && r.getSubRelType().equalsIgnoreCase(ModelConstants.AFFILIATION));
|
||||||
|
|
||||||
Dataset<Organization> organizations = Utils.readPath(spark, inputPath + "/organization", Organization.class);
|
Dataset<Organization> organizations = Utils
|
||||||
|
.readPath(spark, inputPath + "/organization", Organization.class)
|
||||||
|
.filter((FilterFunction<Organization>) o -> !o.getDataInfo().getDeletedbyinference());
|
||||||
|
|
||||||
Dataset<ResultOrganizations> resultOrganization = relations
|
Dataset<ResultOrganizations> resultOrganization = relations
|
||||||
.joinWith(organizations, relations.col("source").equalTo(organizations.col("id")), "left")
|
.joinWith(organizations, relations.col("source").equalTo(organizations.col("id")))
|
||||||
.map((MapFunction<Tuple2<Relation, Organization>, ResultOrganizations>) t2 -> {
|
.map((MapFunction<Tuple2<Relation, Organization>, ResultOrganizations>) t2 -> {
|
||||||
|
ResultOrganizations rOrg = new ResultOrganizations();
|
||||||
|
|
||||||
if (t2._2() != null) {
|
if (t2._2() != null) {
|
||||||
ResultOrganizations rOrg = new ResultOrganizations();
|
|
||||||
rOrg.setResultId(t2._1().getTarget());
|
rOrg.setResultId(t2._1().getTarget());
|
||||||
eu.dnetlib.dhp.eosc.model.Organization org = new eu.dnetlib.dhp.eosc.model.Organization();
|
Affiliation org = new Affiliation();
|
||||||
org.setId(t2._2().getId());
|
org.setId(t2._2().getId());
|
||||||
if (Optional.ofNullable(t2._2().getLegalname()).isPresent()) {
|
if (Optional.ofNullable(t2._2().getLegalname()).isPresent()) {
|
||||||
org.setName(t2._2().getLegalname().getValue());
|
org.setName(t2._2().getLegalname().getValue());
|
||||||
|
@ -115,41 +196,149 @@ public class ExtendEoscResultWithOrganizationStep2 implements Serializable {
|
||||||
rOrg.setAffiliation(org);
|
rOrg.setAffiliation(org);
|
||||||
return rOrg;
|
return rOrg;
|
||||||
}
|
}
|
||||||
return null;
|
return rOrg;
|
||||||
|
|
||||||
}, Encoders.bean(ResultOrganizations.class))
|
}, Encoders.bean(ResultOrganizations.class))
|
||||||
.filter(Objects::nonNull);
|
.filter((FilterFunction<ResultOrganizations>) ro -> ro.getResultId() != null);
|
||||||
|
|
||||||
|
// resultOrganization.count();
|
||||||
|
|
||||||
results
|
results
|
||||||
.joinWith(resultOrganization, results.col("id").equalTo(resultOrganization.col("resultId")), "left")
|
.joinWith(resultOrganization, results.col("id").equalTo(resultOrganization.col("resultId")), "left")
|
||||||
.groupByKey(
|
.groupByKey(
|
||||||
(MapFunction<Tuple2<Result, ResultOrganizations>, String>) t2 -> t2._1().getId(), Encoders.STRING())
|
(MapFunction<Tuple2<Result, ResultOrganizations>, String>) t2 -> t2._1().getId(), Encoders.STRING())
|
||||||
.mapGroups(
|
.mapGroups(
|
||||||
(MapGroupsFunction<String, Tuple2<Result, ResultOrganizations>, Result>) (s, it) -> {
|
(MapGroupsFunction<String, Tuple2<Result, ResultOrganizations>, Result>) (s, it) -> addAffiliation(it),
|
||||||
Tuple2<Result, ResultOrganizations> first = it.next();
|
Encoders.bean(Result.class))
|
||||||
if (first._2() == null) {
|
|
||||||
return first._1();
|
|
||||||
}
|
|
||||||
Result ret = first._1();
|
|
||||||
List<eu.dnetlib.dhp.eosc.model.Organization> affiliation = new ArrayList<>();
|
|
||||||
Set<String> alreadyInsertedAffiliations = new HashSet<>();
|
|
||||||
affiliation.add(first._2().getAffiliation());
|
|
||||||
alreadyInsertedAffiliations.add(first._2().getAffiliation().getId());
|
|
||||||
it.forEachRemaining(res -> {
|
|
||||||
if (!alreadyInsertedAffiliations.contains(res._2().getAffiliation().getId())) {
|
|
||||||
affiliation.add(res._2().getAffiliation());
|
|
||||||
alreadyInsertedAffiliations.add(res._2().getAffiliation().getId());
|
|
||||||
}
|
|
||||||
|
|
||||||
});
|
|
||||||
ret.setAffiliation(affiliation);
|
|
||||||
return ret;
|
|
||||||
}, Encoders.bean(Result.class))
|
|
||||||
.write()
|
.write()
|
||||||
.mode(SaveMode.Overwrite)
|
.mode(SaveMode.Overwrite)
|
||||||
.option("compression", "gzip")
|
.option("compression", "gzip")
|
||||||
.json(outputPath);
|
.json(workingPath + resultType + "extendedaffiliation");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
private static ResultOrganizations getResultOrganizations(Tuple2<Relation, Organization> t2) {
|
||||||
|
if (t2._2() != null) {
|
||||||
|
Organization organization = t2._2();
|
||||||
|
ResultOrganizations rOrg = new ResultOrganizations();
|
||||||
|
rOrg.setResultId(t2._1().getTarget());
|
||||||
|
Affiliation org = new Affiliation();
|
||||||
|
org.setId(organization.getId());
|
||||||
|
if (Optional.ofNullable(organization.getLegalname()).isPresent()) {
|
||||||
|
org.setName(organization.getLegalname().getValue());
|
||||||
|
} else {
|
||||||
|
org.setName("");
|
||||||
|
}
|
||||||
|
HashMap<String, Set<String>> organizationPids = new HashMap<>();
|
||||||
|
if (Optional.ofNullable(organization.getPid()).isPresent())
|
||||||
|
organization.getPid().forEach(p -> {
|
||||||
|
if (!organizationPids.containsKey(p.getQualifier().getClassid()))
|
||||||
|
organizationPids.put(p.getQualifier().getClassid(), new HashSet<>());
|
||||||
|
organizationPids.get(p.getQualifier().getClassid()).add(p.getValue());
|
||||||
|
});
|
||||||
|
List<OrganizationPid> pids = new ArrayList<>();
|
||||||
|
for (String key : organizationPids.keySet()) {
|
||||||
|
for (String value : organizationPids.get(key)) {
|
||||||
|
OrganizationPid pid = new OrganizationPid();
|
||||||
|
pid.setValue(value);
|
||||||
|
pid.setType(key);
|
||||||
|
pids.add(pid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
org.setPid(pids);
|
||||||
|
rOrg.setAffiliation(org);
|
||||||
|
return rOrg;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Result addAffiliation(Iterator<Tuple2<Result, ResultOrganizations>> it) {
|
||||||
|
Tuple2<Result, ResultOrganizations> first = it.next();
|
||||||
|
if (first._2() == null) {
|
||||||
|
return first._1();
|
||||||
|
}
|
||||||
|
Result ret = first._1();
|
||||||
|
List<Affiliation> affiliation = new ArrayList<>();
|
||||||
|
Set<String> alreadyInsertedAffiliations = new HashSet<>();
|
||||||
|
affiliation.add(first._2().getAffiliation());
|
||||||
|
alreadyInsertedAffiliations.add(first._2().getAffiliation().getId());
|
||||||
|
it.forEachRemaining(res -> {
|
||||||
|
if (!alreadyInsertedAffiliations.contains(res._2().getAffiliation().getId())) {
|
||||||
|
affiliation.add(res._2().getAffiliation());
|
||||||
|
alreadyInsertedAffiliations.add(res._2().getAffiliation().getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
ret.setAffiliation(affiliation);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static eu.dnetlib.dhp.eosc.model.Organization mapOrganization(Organization org) {
|
||||||
|
|
||||||
|
if (isToBeRemovedOrg(org))
|
||||||
|
return null;
|
||||||
|
|
||||||
|
eu.dnetlib.dhp.eosc.model.Organization organization = new eu.dnetlib.dhp.eosc.model.Organization();
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getLegalshortname())
|
||||||
|
.ifPresent(value -> organization.setLegalshortname(value.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getLegalname())
|
||||||
|
.ifPresent(value -> organization.setLegalname(value.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getWebsiteurl())
|
||||||
|
.ifPresent(value -> organization.setWebsiteurl(value.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getAlternativeNames())
|
||||||
|
.ifPresent(
|
||||||
|
value -> organization
|
||||||
|
.setAlternativenames(
|
||||||
|
value
|
||||||
|
.stream()
|
||||||
|
.map(v -> v.getValue())
|
||||||
|
.collect(Collectors.toList())));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getCountry())
|
||||||
|
.ifPresent(
|
||||||
|
value -> {
|
||||||
|
if (!value.getClassid().equals(UNKNOWN)) {
|
||||||
|
organization
|
||||||
|
.setCountry(
|
||||||
|
Country.newInstance(value.getClassid(), value.getClassname()));
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getId())
|
||||||
|
.ifPresent(value -> organization.setId(value));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(org.getPid())
|
||||||
|
.ifPresent(
|
||||||
|
value -> organization
|
||||||
|
.setPid(
|
||||||
|
value
|
||||||
|
.stream()
|
||||||
|
.map(p -> OrganizationPid.newInstance(p.getQualifier().getClassid(), p.getValue()))
|
||||||
|
.collect(Collectors.toList())));
|
||||||
|
|
||||||
|
return organization;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isToBeRemovedOrg(Organization org) {
|
||||||
|
if (Boolean.TRUE.equals(org.getDataInfo().getDeletedbyinference()))
|
||||||
|
return true;
|
||||||
|
if (!Optional.ofNullable(org.getLegalname()).isPresent()
|
||||||
|
&& !Optional.ofNullable(org.getLegalshortname()).isPresent())
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,9 +2,8 @@
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.eosc.model.Organization;
|
import eu.dnetlib.dhp.eosc.model.Affiliation;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author miriam.baglioni
|
* @author miriam.baglioni
|
||||||
|
@ -12,7 +11,7 @@ import eu.dnetlib.dhp.eosc.model.Organization;
|
||||||
*/
|
*/
|
||||||
public class ResultOrganizations implements Serializable {
|
public class ResultOrganizations implements Serializable {
|
||||||
private String resultId;
|
private String resultId;
|
||||||
private Organization affiliation;
|
private Affiliation affiliation;
|
||||||
|
|
||||||
public String getResultId() {
|
public String getResultId() {
|
||||||
return resultId;
|
return resultId;
|
||||||
|
@ -22,11 +21,11 @@ public class ResultOrganizations implements Serializable {
|
||||||
this.resultId = resultId;
|
this.resultId = resultId;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Organization getAffiliation() {
|
public Affiliation getAffiliation() {
|
||||||
return affiliation;
|
return affiliation;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setAffiliation(Organization affiliation) {
|
public void setAffiliation(Affiliation affiliation) {
|
||||||
this.affiliation = affiliation;
|
this.affiliation = affiliation;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,11 +4,11 @@ package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.eosc.model.Project;
|
import eu.dnetlib.dhp.eosc.model.ProjectSummary;
|
||||||
|
|
||||||
public class ResultProject implements Serializable {
|
public class ResultProject implements Serializable {
|
||||||
private String resultId;
|
private String resultId;
|
||||||
private List<Project> projectsList;
|
private List<ProjectSummary> projectsList;
|
||||||
|
|
||||||
public String getResultId() {
|
public String getResultId() {
|
||||||
return resultId;
|
return resultId;
|
||||||
|
@ -18,11 +18,11 @@ public class ResultProject implements Serializable {
|
||||||
this.resultId = resultId;
|
this.resultId = resultId;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Project> getProjectsList() {
|
public List<ProjectSummary> getProjectsList() {
|
||||||
return projectsList;
|
return projectsList;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setProjectsList(List<Project> projectsList) {
|
public void setProjectsList(List<ProjectSummary> projectsList) {
|
||||||
this.projectsList = projectsList;
|
this.projectsList = projectsList;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,16 +13,14 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.dom4j.DocumentException;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.xml.sax.SAXException;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class connects with the IS related to the isLookUpUrl got as parameter. It saves the information about the
|
* This class connects with the community APIs for production. It saves the information about the
|
||||||
* context that will guide the dump of the results. The information saved is a HashMap. The key is the id of a community
|
* context that will guide the dump of the results. The information saved is a HashMap. The key is the id of a community
|
||||||
* - research infrastructure/initiative , the value is the label of the research community - research
|
* - research infrastructure/initiative , the value is the label of the research community - research
|
||||||
* infrastructure/initiative.
|
* infrastructure/initiative.
|
||||||
|
@ -31,11 +29,11 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
||||||
public class SaveCommunityMap implements Serializable {
|
public class SaveCommunityMap implements Serializable {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(SaveCommunityMap.class);
|
private static final Logger log = LoggerFactory.getLogger(SaveCommunityMap.class);
|
||||||
private final transient QueryInformationSystem queryInformationSystem;
|
private final transient UtilCommunityAPI queryInformationSystem;
|
||||||
|
|
||||||
private final transient BufferedWriter writer;
|
private final transient BufferedWriter writer;
|
||||||
|
|
||||||
public SaveCommunityMap(String hdfsPath, String hdfsNameNode, String isLookUpUrl) throws IOException {
|
public SaveCommunityMap(String hdfsPath, String hdfsNameNode) throws IOException {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
conf.set("fs.defaultFS", hdfsNameNode);
|
conf.set("fs.defaultFS", hdfsNameNode);
|
||||||
FileSystem fileSystem = FileSystem.get(conf);
|
FileSystem fileSystem = FileSystem.get(conf);
|
||||||
|
@ -45,8 +43,7 @@ public class SaveCommunityMap implements Serializable {
|
||||||
fileSystem.delete(hdfsWritePath, true);
|
fileSystem.delete(hdfsWritePath, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
queryInformationSystem = new QueryInformationSystem();
|
queryInformationSystem = new UtilCommunityAPI();
|
||||||
queryInformationSystem.setIsLookUp(Utils.getIsLookUpService(isLookUpUrl));
|
|
||||||
|
|
||||||
FSDataOutputStream fos = fileSystem.create(hdfsWritePath);
|
FSDataOutputStream fos = fileSystem.create(hdfsWritePath);
|
||||||
writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8));
|
writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8));
|
||||||
|
@ -68,19 +65,23 @@ public class SaveCommunityMap implements Serializable {
|
||||||
final String outputPath = parser.get("outputPath");
|
final String outputPath = parser.get("outputPath");
|
||||||
log.info("outputPath: {}", outputPath);
|
log.info("outputPath: {}", outputPath);
|
||||||
|
|
||||||
final String isLookUpUrl = parser.get("isLookUpUrl");
|
final Boolean singleCommunity = Optional
|
||||||
log.info("isLookUpUrl: {}", isLookUpUrl);
|
.ofNullable(parser.get("singleDeposition"))
|
||||||
|
.map(Boolean::valueOf)
|
||||||
|
.orElse(false);
|
||||||
|
|
||||||
final SaveCommunityMap scm = new SaveCommunityMap(outputPath, nameNode, isLookUpUrl);
|
final String community_id = Optional.ofNullable(parser.get("communityId")).orElse(null);
|
||||||
|
|
||||||
scm.saveCommunityMap();
|
final SaveCommunityMap scm = new SaveCommunityMap(outputPath, nameNode);
|
||||||
|
|
||||||
|
scm.saveCommunityMap(singleCommunity, community_id);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveCommunityMap()
|
private void saveCommunityMap(boolean singleCommunity, String communityId)
|
||||||
throws ISLookUpException, IOException, DocumentException, SAXException {
|
throws IOException {
|
||||||
final String communityMapString = Utils.OBJECT_MAPPER
|
final String communityMapString = Utils.OBJECT_MAPPER
|
||||||
.writeValueAsString(queryInformationSystem.getCommunityMap());
|
.writeValueAsString(queryInformationSystem.getCommunityMap(singleCommunity, communityId));
|
||||||
log.info("communityMap {} ", communityMapString);
|
log.info("communityMap {} ", communityMapString);
|
||||||
writer
|
writer
|
||||||
.write(
|
.write(
|
||||||
|
|
|
@ -0,0 +1,167 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
|
|
||||||
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.function.FilterFunction;
|
||||||
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
|
import org.apache.spark.api.java.function.MapGroupsFunction;
|
||||||
|
import org.apache.spark.sql.Dataset;
|
||||||
|
import org.apache.spark.sql.Encoders;
|
||||||
|
import org.apache.spark.sql.SaveMode;
|
||||||
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Organization;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Project;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Provenance;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.RelType;
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Relation;
|
||||||
|
import scala.Tuple2;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 12/01/23
|
||||||
|
*/
|
||||||
|
public class SparkDumpOrganizationProject implements Serializable {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(SparkDumpOrganizationProject.class);
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
String jsonConfiguration = IOUtils
|
||||||
|
.toString(
|
||||||
|
SparkDumpOrganizationProject.class
|
||||||
|
.getResourceAsStream(
|
||||||
|
"/eu/dnetlib/dhp/oa/graph/dump/eosc_organizationprojectrelations_parameters.json"));
|
||||||
|
|
||||||
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
||||||
|
parser.parseArgument(args);
|
||||||
|
|
||||||
|
Boolean isSparkSessionManaged = Optional
|
||||||
|
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||||
|
.map(Boolean::valueOf)
|
||||||
|
.orElse(Boolean.TRUE);
|
||||||
|
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||||
|
|
||||||
|
final String inputPath = parser.get("sourcePath");
|
||||||
|
log.info("inputPath: {}", inputPath);
|
||||||
|
|
||||||
|
final String outputPath = parser.get("outputPath");
|
||||||
|
log.info("outputPath: {}", outputPath);
|
||||||
|
|
||||||
|
final String workingPath = parser.get("workingPath");
|
||||||
|
log.info("workingPath: {}", workingPath);
|
||||||
|
|
||||||
|
SparkConf conf = new SparkConf();
|
||||||
|
|
||||||
|
runWithSparkSession(
|
||||||
|
conf,
|
||||||
|
isSparkSessionManaged,
|
||||||
|
spark -> {
|
||||||
|
Utils.removeOutputDir(spark, outputPath + "/organizationProject");
|
||||||
|
dumpRelation(spark, inputPath, outputPath, workingPath);
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void dumpRelation(SparkSession spark, String inputPath, String outputPath, String workingPath) {
|
||||||
|
Dataset<Organization> organization = Utils
|
||||||
|
.readPath(spark, workingPath + "publicationorganization", Organization.class)
|
||||||
|
.union(Utils.readPath(spark, workingPath + "datasetorganization", Organization.class))
|
||||||
|
.union(Utils.readPath(spark, workingPath + "softwareorganization", Organization.class))
|
||||||
|
.union(Utils.readPath(spark, workingPath + "otherresearchproductorganization", Organization.class))
|
||||||
|
.groupByKey((MapFunction<Organization, String>) o -> o.getId(), Encoders.STRING())
|
||||||
|
.mapGroups(
|
||||||
|
(MapGroupsFunction<String, Organization, Organization>) (k, v) -> v.next(),
|
||||||
|
Encoders.bean(Organization.class));
|
||||||
|
|
||||||
|
Dataset<Project> project = Utils
|
||||||
|
.readPath(spark, workingPath + "publicationproject", Project.class)
|
||||||
|
.union(Utils.readPath(spark, workingPath + "datasetproject", Project.class))
|
||||||
|
.union(Utils.readPath(spark, workingPath + "softwareproject", Project.class))
|
||||||
|
.union(Utils.readPath(spark, workingPath + "otherresearchproductproject", Project.class))
|
||||||
|
.groupByKey((MapFunction<Project, String>) o -> o.getId(), Encoders.STRING())
|
||||||
|
.mapGroups((MapGroupsFunction<String, Project, Project>) (k, v) -> v.next(), Encoders.bean(Project.class));
|
||||||
|
|
||||||
|
organization
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(outputPath + "organization");
|
||||||
|
|
||||||
|
project
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(outputPath + "project");
|
||||||
|
|
||||||
|
Dataset<Relation> relation = Utils
|
||||||
|
.readPath(spark, inputPath + "/relation", Relation.class)
|
||||||
|
.filter(
|
||||||
|
(FilterFunction<Relation>) r -> !r.getDataInfo().getDeletedbyinference()
|
||||||
|
&& r.getRelClass().equalsIgnoreCase(ModelConstants.IS_PARTICIPANT));
|
||||||
|
|
||||||
|
Dataset<Relation> eoscOrgs = relation
|
||||||
|
.joinWith(organization, relation.col("source").equalTo(organization.col("id")))
|
||||||
|
.map((MapFunction<Tuple2<Relation, Organization>, Relation>) t2 -> t2._1(), Encoders.bean(Relation.class));
|
||||||
|
|
||||||
|
eoscOrgs
|
||||||
|
.joinWith(project, eoscOrgs.col("target").equalTo(project.col("id")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<Relation, Project>, eu.dnetlib.dhp.eosc.model.Relation>) t2 -> eu.dnetlib.dhp.eosc.model.Relation
|
||||||
|
.newInstance(t2._1().getSource(), t2._1().getTarget()),
|
||||||
|
Encoders.bean(eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(outputPath + "organizationProject");
|
||||||
|
|
||||||
|
Utils
|
||||||
|
.readPath(spark, workingPath + "publicationresultOrganization", eu.dnetlib.dhp.eosc.model.Relation.class)
|
||||||
|
.union(
|
||||||
|
Utils
|
||||||
|
.readPath(
|
||||||
|
spark, workingPath + "datasetresultOrganization", eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.union(
|
||||||
|
Utils
|
||||||
|
.readPath(
|
||||||
|
spark, workingPath + "softwareresultOrganization", eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.union(
|
||||||
|
Utils
|
||||||
|
.readPath(
|
||||||
|
spark, workingPath + "otherresearchproductresultOrganization",
|
||||||
|
eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(outputPath + "resultOrganization");
|
||||||
|
|
||||||
|
Utils
|
||||||
|
.readPath(spark, workingPath + "publicationresultProject", eu.dnetlib.dhp.eosc.model.Relation.class)
|
||||||
|
.union(
|
||||||
|
Utils.readPath(spark, workingPath + "datasetresultProject", eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.union(
|
||||||
|
Utils.readPath(spark, workingPath + "softwareresultProject", eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.union(
|
||||||
|
Utils
|
||||||
|
.readPath(
|
||||||
|
spark, workingPath + "otherresearchproductresultProject",
|
||||||
|
eu.dnetlib.dhp.eosc.model.Relation.class))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(outputPath + "resultProject");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,115 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
|
||||||
|
|
||||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
|
||||||
import org.apache.spark.SparkConf;
|
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
|
||||||
import org.apache.spark.sql.Dataset;
|
|
||||||
import org.apache.spark.sql.Encoders;
|
|
||||||
import org.apache.spark.sql.SaveMode;
|
|
||||||
import org.apache.spark.sql.SparkSession;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
||||||
import eu.dnetlib.dhp.eosc.model.Provenance;
|
|
||||||
import eu.dnetlib.dhp.eosc.model.RelType;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Relation;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author miriam.baglioni
|
|
||||||
* @Date 12/01/23
|
|
||||||
*/
|
|
||||||
public class SparkDumpRelation implements Serializable {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(SparkDumpRelation.class);
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
|
||||||
String jsonConfiguration = IOUtils
|
|
||||||
.toString(
|
|
||||||
SparkDumpRelation.class
|
|
||||||
.getResourceAsStream(
|
|
||||||
"/eu/dnetlib/dhp/oa/graph/dump/input_relationdump_parameters.json"));
|
|
||||||
|
|
||||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration);
|
|
||||||
parser.parseArgument(args);
|
|
||||||
|
|
||||||
Boolean isSparkSessionManaged = Optional
|
|
||||||
.ofNullable(parser.get("isSparkSessionManaged"))
|
|
||||||
.map(Boolean::valueOf)
|
|
||||||
.orElse(Boolean.TRUE);
|
|
||||||
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
|
||||||
|
|
||||||
final String inputPath = parser.get("sourcePath");
|
|
||||||
log.info("inputPath: {}", inputPath);
|
|
||||||
|
|
||||||
final String outputPath = parser.get("outputPath");
|
|
||||||
log.info("outputPath: {}", outputPath);
|
|
||||||
|
|
||||||
SparkConf conf = new SparkConf();
|
|
||||||
|
|
||||||
runWithSparkSession(
|
|
||||||
conf,
|
|
||||||
isSparkSessionManaged,
|
|
||||||
spark -> {
|
|
||||||
Utils.removeOutputDir(spark, outputPath);
|
|
||||||
dumpRelation(spark, inputPath, outputPath);
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void dumpRelation(SparkSession spark, String inputPath, String outputPath) {
|
|
||||||
Dataset<Relation> relations = Utils.readPath(spark, inputPath, Relation.class);
|
|
||||||
relations
|
|
||||||
|
|
||||||
.map((MapFunction<Relation, eu.dnetlib.dhp.eosc.model.Relation>) relation -> {
|
|
||||||
eu.dnetlib.dhp.eosc.model.Relation relNew = new eu.dnetlib.dhp.eosc.model.Relation();
|
|
||||||
relNew
|
|
||||||
.setSource(
|
|
||||||
|
|
||||||
relation.getSource());
|
|
||||||
|
|
||||||
relNew
|
|
||||||
.setTarget(
|
|
||||||
|
|
||||||
relation.getTarget());
|
|
||||||
|
|
||||||
relNew
|
|
||||||
.setReltype(
|
|
||||||
RelType
|
|
||||||
.newInstance(
|
|
||||||
relation.getRelClass(),
|
|
||||||
relation.getSubRelType()));
|
|
||||||
|
|
||||||
Optional<DataInfo> odInfo = Optional.ofNullable(relation.getDataInfo());
|
|
||||||
if (odInfo.isPresent()) {
|
|
||||||
DataInfo dInfo = odInfo.get();
|
|
||||||
if (Optional.ofNullable(dInfo.getProvenanceaction()).isPresent() &&
|
|
||||||
Optional.ofNullable(dInfo.getProvenanceaction().getClassname()).isPresent()) {
|
|
||||||
relNew
|
|
||||||
.setProvenance(
|
|
||||||
Provenance
|
|
||||||
.newInstance(
|
|
||||||
dInfo.getProvenanceaction().getClassname(),
|
|
||||||
dInfo.getTrust()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return relNew;
|
|
||||||
|
|
||||||
}, Encoders.bean(eu.dnetlib.dhp.eosc.model.Relation.class))
|
|
||||||
.write()
|
|
||||||
.option("compression", "gzip")
|
|
||||||
.mode(SaveMode.Append)
|
|
||||||
.json(outputPath);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -26,8 +26,8 @@ import org.slf4j.LoggerFactory;
|
||||||
import org.xml.sax.SAXException;
|
import org.xml.sax.SAXException;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.dhp.eosc.model.Funder;
|
import eu.dnetlib.dhp.eosc.model.FunderShort;
|
||||||
import eu.dnetlib.dhp.eosc.model.Project;
|
import eu.dnetlib.dhp.eosc.model.ProjectSummary;
|
||||||
import eu.dnetlib.dhp.eosc.model.Provenance;
|
import eu.dnetlib.dhp.eosc.model.Provenance;
|
||||||
import eu.dnetlib.dhp.eosc.model.Validated;
|
import eu.dnetlib.dhp.eosc.model.Validated;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
@ -103,9 +103,9 @@ public class SparkPrepareResultProject implements Serializable {
|
||||||
rp.setResultId(s);
|
rp.setResultId(s);
|
||||||
eu.dnetlib.dhp.schema.oaf.Project p = first._1();
|
eu.dnetlib.dhp.schema.oaf.Project p = first._1();
|
||||||
projectSet.add(p.getId());
|
projectSet.add(p.getId());
|
||||||
Project ps = getProject(p, first._2);
|
ProjectSummary ps = getProject(p, first._2);
|
||||||
|
|
||||||
List<Project> projList = new ArrayList<>();
|
List<ProjectSummary> projList = new ArrayList<>();
|
||||||
projList.add(ps);
|
projList.add(ps);
|
||||||
rp.setProjectsList(projList);
|
rp.setProjectsList(projList);
|
||||||
it.forEachRemaining(c -> {
|
it.forEachRemaining(c -> {
|
||||||
|
@ -132,8 +132,8 @@ public class SparkPrepareResultProject implements Serializable {
|
||||||
.json(outputPath);
|
.json(outputPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Project getProject(eu.dnetlib.dhp.schema.oaf.Project op, Relation relation) {
|
private static ProjectSummary getProject(eu.dnetlib.dhp.schema.oaf.Project op, Relation relation) {
|
||||||
Project p = Project
|
ProjectSummary p = ProjectSummary
|
||||||
.newInstance(
|
.newInstance(
|
||||||
op.getId(),
|
op.getId(),
|
||||||
op.getCode().getValue(),
|
op.getCode().getValue(),
|
||||||
|
@ -148,7 +148,7 @@ public class SparkPrepareResultProject implements Serializable {
|
||||||
Optional
|
Optional
|
||||||
.ofNullable(op.getFundingtree())
|
.ofNullable(op.getFundingtree())
|
||||||
.map(value -> {
|
.map(value -> {
|
||||||
List<Funder> tmp = value
|
List<FunderShort> tmp = value
|
||||||
.stream()
|
.stream()
|
||||||
.map(ft -> getFunder(ft.getValue()))
|
.map(ft -> getFunder(ft.getValue()))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
@ -174,8 +174,8 @@ public class SparkPrepareResultProject implements Serializable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Funder getFunder(String fundingtree) {
|
private static FunderShort getFunder(String fundingtree) {
|
||||||
final Funder f = new Funder();
|
final FunderShort f = new FunderShort();
|
||||||
final Document doc;
|
final Document doc;
|
||||||
try {
|
try {
|
||||||
final SAXReader reader = new SAXReader();
|
final SAXReader reader = new SAXReader();
|
||||||
|
|
|
@ -80,14 +80,12 @@ public class SparkSelectRelation implements Serializable {
|
||||||
|
|
||||||
Dataset<Tuple2<String, String>> resultIds = Utils
|
Dataset<Tuple2<String, String>> resultIds = Utils
|
||||||
.readPath(spark, outputPath + "/publication", Result.class)
|
.readPath(spark, outputPath + "/publication", Result.class)
|
||||||
|
|
||||||
.map(
|
.map(
|
||||||
(MapFunction<Result, Tuple2<String, String>>) p -> new Tuple2<>(p.getId(), p.getType()),
|
(MapFunction<Result, Tuple2<String, String>>) p -> new Tuple2<>(p.getId(), p.getType()),
|
||||||
Encoders.tuple(Encoders.STRING(), Encoders.STRING()))
|
Encoders.tuple(Encoders.STRING(), Encoders.STRING()))
|
||||||
.union(
|
.union(
|
||||||
Utils
|
Utils
|
||||||
.readPath(spark, outputPath + "/dataset", Result.class)
|
.readPath(spark, outputPath + "/dataset", Result.class)
|
||||||
|
|
||||||
.map(
|
.map(
|
||||||
(MapFunction<Result, Tuple2<String, String>>) d -> new Tuple2<>(d.getId(), d.getType()),
|
(MapFunction<Result, Tuple2<String, String>>) d -> new Tuple2<>(d.getId(), d.getType()),
|
||||||
Encoders.tuple(Encoders.STRING(), Encoders.STRING())))
|
Encoders.tuple(Encoders.STRING(), Encoders.STRING())))
|
||||||
|
|
|
@ -4,23 +4,33 @@ package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
import java.io.StringReader;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.function.FlatMapFunction;
|
||||||
|
import org.apache.spark.api.java.function.ForeachFunction;
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
import org.apache.spark.sql.Dataset;
|
import org.apache.spark.sql.*;
|
||||||
import org.apache.spark.sql.Encoders;
|
import org.dom4j.Document;
|
||||||
import org.apache.spark.sql.SaveMode;
|
import org.dom4j.DocumentException;
|
||||||
import org.apache.spark.sql.SparkSession;
|
import org.dom4j.Node;
|
||||||
|
import org.dom4j.io.SAXReader;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||||
import eu.dnetlib.dhp.eosc.model.Result;
|
import eu.dnetlib.dhp.eosc.model.*;
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.Constants;
|
import eu.dnetlib.dhp.oa.graph.dump.Constants;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Field;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Project;
|
||||||
|
import scala.Array;
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
|
||||||
public class SparkUpdateProjectInfo implements Serializable {
|
public class SparkUpdateProjectInfo implements Serializable {
|
||||||
|
@ -47,12 +57,18 @@ public class SparkUpdateProjectInfo implements Serializable {
|
||||||
final String inputPath = parser.get("sourcePath");
|
final String inputPath = parser.get("sourcePath");
|
||||||
log.info("inputPath: {}", inputPath);
|
log.info("inputPath: {}", inputPath);
|
||||||
|
|
||||||
final String outputPath = parser.get("outputPath");
|
final String workingPath = parser.get("workingPath");
|
||||||
log.info("outputPath: {}", outputPath);
|
log.info("workingPath: {}", workingPath);
|
||||||
|
|
||||||
final String preparedInfoPath = parser.get("preparedInfoPath");
|
final String preparedInfoPath = parser.get("preparedInfoPath");
|
||||||
log.info("preparedInfoPath: {}", preparedInfoPath);
|
log.info("preparedInfoPath: {}", preparedInfoPath);
|
||||||
|
|
||||||
|
final String outputPath = parser.get("outputPath");
|
||||||
|
log.info("outputPath: {}", outputPath);
|
||||||
|
|
||||||
|
final String resultType = parser.get("resultType");
|
||||||
|
log.info("resultType: {}", resultType);
|
||||||
|
|
||||||
final String dumpType = Optional
|
final String dumpType = Optional
|
||||||
.ofNullable(parser.get("dumpType"))
|
.ofNullable(parser.get("dumpType"))
|
||||||
.orElse(Constants.DUMPTYPE.COMMUNITY.getType());
|
.orElse(Constants.DUMPTYPE.COMMUNITY.getType());
|
||||||
|
@ -64,19 +80,20 @@ public class SparkUpdateProjectInfo implements Serializable {
|
||||||
conf,
|
conf,
|
||||||
isSparkSessionManaged,
|
isSparkSessionManaged,
|
||||||
spark -> {
|
spark -> {
|
||||||
Utils.removeOutputDir(spark, outputPath);
|
Utils.removeOutputDir(spark, workingPath + resultType + "extendedproject");
|
||||||
extend(spark, inputPath, outputPath, preparedInfoPath);
|
extend(spark, inputPath, workingPath, preparedInfoPath, outputPath, resultType);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void extend(
|
private static void extend(
|
||||||
SparkSession spark,
|
SparkSession spark,
|
||||||
String inputPath,
|
String inputPath,
|
||||||
|
String workingPath,
|
||||||
|
String preparedInfoPath,
|
||||||
String outputPath,
|
String outputPath,
|
||||||
String preparedInfoPath) {
|
String resultType) {
|
||||||
|
|
||||||
Dataset<Result> result = Utils.readPath(spark, inputPath, Result.class);
|
|
||||||
|
|
||||||
|
Dataset<Result> result = Utils.readPath(spark, workingPath + resultType + "extendedaffiliation", Result.class);
|
||||||
Dataset<ResultProject> resultProject = Utils.readPath(spark, preparedInfoPath, ResultProject.class);
|
Dataset<ResultProject> resultProject = Utils.readPath(spark, preparedInfoPath, ResultProject.class);
|
||||||
result
|
result
|
||||||
.joinWith(
|
.joinWith(
|
||||||
|
@ -90,7 +107,209 @@ public class SparkUpdateProjectInfo implements Serializable {
|
||||||
.write()
|
.write()
|
||||||
.option("compression", "gzip")
|
.option("compression", "gzip")
|
||||||
.mode(SaveMode.Append)
|
.mode(SaveMode.Append)
|
||||||
.json(outputPath);
|
.json(workingPath + resultType + "extendedproject");
|
||||||
|
|
||||||
|
Dataset<Project> project = Utils.readPath(spark, inputPath + "/project", Project.class);
|
||||||
|
|
||||||
|
Dataset<String> projectIds = result
|
||||||
|
.joinWith(resultProject, result.col("id").equalTo(resultProject.col("resultId")))
|
||||||
|
.flatMap(
|
||||||
|
(FlatMapFunction<Tuple2<Result, ResultProject>, String>) t2 -> t2
|
||||||
|
._2()
|
||||||
|
.getProjectsList()
|
||||||
|
.stream()
|
||||||
|
.map(p -> p.getId())
|
||||||
|
.collect(Collectors.toList())
|
||||||
|
.iterator(),
|
||||||
|
Encoders.STRING())
|
||||||
|
.distinct();
|
||||||
|
|
||||||
|
projectIds
|
||||||
|
.joinWith(project, projectIds.col("value").equalTo(project.col("id")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<String, Project>, eu.dnetlib.dhp.eosc.model.Project>) t2 -> mapProject(t2._2()),
|
||||||
|
Encoders.bean(eu.dnetlib.dhp.eosc.model.Project.class))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(workingPath + resultType + "project");
|
||||||
|
|
||||||
|
result
|
||||||
|
.joinWith(
|
||||||
|
resultProject, result.col("id").equalTo(resultProject.col("resultId")))
|
||||||
|
.map(
|
||||||
|
(MapFunction<Tuple2<Result, ResultProject>, ResultProject>) t2 -> t2._2(),
|
||||||
|
Encoders.bean(ResultProject.class))
|
||||||
|
.flatMap(
|
||||||
|
(FlatMapFunction<ResultProject, Relation>) rp -> rp
|
||||||
|
.getProjectsList()
|
||||||
|
.stream()
|
||||||
|
.map(p -> Relation.newInstance(rp.getResultId(), p.getId()))
|
||||||
|
.collect(Collectors.toList())
|
||||||
|
.iterator(),
|
||||||
|
Encoders.bean(Relation.class))
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.option("compression", "gzip")
|
||||||
|
.json(workingPath + resultType + "resultProject");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static eu.dnetlib.dhp.eosc.model.Project mapProject(eu.dnetlib.dhp.schema.oaf.Project p)
|
||||||
|
throws DocumentException {
|
||||||
|
if (Boolean.TRUE.equals(p.getDataInfo().getDeletedbyinference()))
|
||||||
|
return null;
|
||||||
|
|
||||||
|
eu.dnetlib.dhp.eosc.model.Project project = new eu.dnetlib.dhp.eosc.model.Project();
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getId())
|
||||||
|
.ifPresent(id -> project.setId(id));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getWebsiteurl())
|
||||||
|
.ifPresent(w -> project.setWebsiteurl(w.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getCode())
|
||||||
|
.ifPresent(code -> project.setCode(code.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getAcronym())
|
||||||
|
.ifPresent(acronynim -> project.setAcronym(acronynim.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getTitle())
|
||||||
|
.ifPresent(title -> project.setTitle(title.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getStartdate())
|
||||||
|
.ifPresent(sdate -> project.setStartdate(sdate.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getEnddate())
|
||||||
|
.ifPresent(edate -> project.setEnddate(edate.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getCallidentifier())
|
||||||
|
.ifPresent(cide -> project.setCallidentifier(cide.getValue()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getKeywords())
|
||||||
|
.ifPresent(key -> project.setKeywords(key.getValue()));
|
||||||
|
|
||||||
|
Optional<Field<String>> omandate = Optional.ofNullable(p.getOamandatepublications());
|
||||||
|
Optional<Field<String>> oecsc39 = Optional.ofNullable(p.getEcsc39());
|
||||||
|
boolean mandate = false;
|
||||||
|
if (omandate.isPresent()) {
|
||||||
|
if (omandate.get().getValue().equals("true")) {
|
||||||
|
mandate = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (oecsc39.isPresent()) {
|
||||||
|
if (oecsc39.get().getValue().equals("true")) {
|
||||||
|
mandate = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
project.setOpenaccessmandateforpublications(mandate);
|
||||||
|
project.setOpenaccessmandatefordataset(false);
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getEcarticle29_3())
|
||||||
|
.ifPresent(oamandate -> project.setOpenaccessmandatefordataset(oamandate.getValue().equals("true")));
|
||||||
|
|
||||||
|
project
|
||||||
|
.setSubject(
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getSubjects())
|
||||||
|
.map(subjs -> subjs.stream().map(s -> s.getValue()).collect(Collectors.toList()))
|
||||||
|
.orElse(new ArrayList<>()));
|
||||||
|
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getSummary())
|
||||||
|
.ifPresent(summary -> project.setSummary(summary.getValue()));
|
||||||
|
|
||||||
|
Optional<Float> ofundedamount = Optional.ofNullable(p.getFundedamount());
|
||||||
|
Optional<Field<String>> ocurrency = Optional.ofNullable(p.getCurrency());
|
||||||
|
Optional<Float> ototalcost = Optional.ofNullable(p.getTotalcost());
|
||||||
|
|
||||||
|
if (ocurrency.isPresent()) {
|
||||||
|
if (ofundedamount.isPresent()) {
|
||||||
|
if (ototalcost.isPresent()) {
|
||||||
|
project
|
||||||
|
.setGranted(
|
||||||
|
Granted.newInstance(ocurrency.get().getValue(), ototalcost.get(), ofundedamount.get()));
|
||||||
|
} else {
|
||||||
|
project.setGranted(Granted.newInstance(ocurrency.get().getValue(), ofundedamount.get()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
project
|
||||||
|
.setH2020programme(
|
||||||
|
Optional
|
||||||
|
.ofNullable(p.getH2020classification())
|
||||||
|
.map(
|
||||||
|
classification -> classification
|
||||||
|
.stream()
|
||||||
|
.map(
|
||||||
|
c -> Programme
|
||||||
|
.newInstance(
|
||||||
|
c.getH2020Programme().getCode(), c.getH2020Programme().getDescription()))
|
||||||
|
.collect(Collectors.toList()))
|
||||||
|
.orElse(new ArrayList<>()));
|
||||||
|
|
||||||
|
Optional<List<Field<String>>> ofundTree = Optional
|
||||||
|
.ofNullable(p.getFundingtree());
|
||||||
|
List<Funder> funList = new ArrayList<>();
|
||||||
|
if (ofundTree.isPresent()) {
|
||||||
|
for (Field<String> fundingtree : ofundTree.get()) {
|
||||||
|
funList.add(getFunder(fundingtree.getValue()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
project.setFunding(funList);
|
||||||
|
|
||||||
|
return project;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Funder getFunder(String fundingtree) throws DocumentException {
|
||||||
|
Funder f = new Funder();
|
||||||
|
final Document doc;
|
||||||
|
|
||||||
|
doc = new SAXReader().read(new StringReader(fundingtree));
|
||||||
|
f.setShortName(((org.dom4j.Node) (doc.selectNodes("//funder/shortname").get(0))).getText());
|
||||||
|
f.setName(((org.dom4j.Node) (doc.selectNodes("//funder/name").get(0))).getText());
|
||||||
|
f.setJurisdiction(((org.dom4j.Node) (doc.selectNodes("//funder/jurisdiction").get(0))).getText());
|
||||||
|
|
||||||
|
String id = "";
|
||||||
|
|
||||||
|
StringBuilder bld = new StringBuilder();
|
||||||
|
|
||||||
|
int level = 0;
|
||||||
|
List<org.dom4j.Node> nodes = doc.selectNodes("//funding_level_" + level);
|
||||||
|
while (!nodes.isEmpty()) {
|
||||||
|
for (org.dom4j.Node n : nodes) {
|
||||||
|
|
||||||
|
List node = n.selectNodes("./id");
|
||||||
|
id = ((org.dom4j.Node) node.get(0)).getText();
|
||||||
|
id = id.substring(id.indexOf("::") + 2);
|
||||||
|
|
||||||
|
node = n.selectNodes("./description");
|
||||||
|
bld.append(((Node) node.get(0)).getText() + " - ");
|
||||||
|
|
||||||
|
}
|
||||||
|
level += 1;
|
||||||
|
nodes = doc.selectNodes("//funding_level_" + level);
|
||||||
|
}
|
||||||
|
String description = bld.toString();
|
||||||
|
if (!id.equals("")) {
|
||||||
|
Fundings fundings = new Fundings();
|
||||||
|
fundings.setId(id);
|
||||||
|
fundings.setDescription(description.substring(0, description.length() - 3).trim());
|
||||||
|
f.setFunding_stream(fundings);
|
||||||
|
}
|
||||||
|
|
||||||
|
return f;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
sourcePath=/tmp/prod_provision/graph/20_graph_blacklisted
|
||||||
|
outputPath=/tmp/miriam/graph_dumps/country_PT
|
||||||
|
country=PT
|
|
@ -0,0 +1,16 @@
|
||||||
|
#PROPERTIES FOR EOSC DUMP
|
||||||
|
sourcePath=/tmp/prod_provision/graph/20_graph_blacklisted
|
||||||
|
outputPath=/tmp/miriam/graph_dumps/eosc_prod_extended
|
||||||
|
#accessToken for the openaire sandbox following
|
||||||
|
accessToken=OzzOsyucEIHxCEfhlpsMo3myEiwpCza3trCRL7ddfGTAK9xXkIP2MbXd6Vg4
|
||||||
|
connectionUrl=https://sandbox.zenodo.org/api/deposit/depositions
|
||||||
|
singleDeposition=false
|
||||||
|
conceptRecordId=1094304
|
||||||
|
depositionType=version
|
||||||
|
metadata=""
|
||||||
|
depositionId=6616871
|
||||||
|
removeSet=merges;isMergedIn
|
||||||
|
postgresURL=jdbc:postgresql://postgresql.services.openaire.eu:5432/dnet_openaireplus
|
||||||
|
postgresUser=dnet
|
||||||
|
postgresPassword=dnetPwd
|
||||||
|
isLookUpUrl=http://services.openaire.eu:8280/is/services/isLookUp?wsdl
|
|
@ -95,7 +95,6 @@
|
||||||
<main-class>eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap</main-class>
|
<main-class>eu.dnetlib.dhp.oa.graph.dump.eosc.SaveCommunityMap</main-class>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
|
<arg>--outputPath</arg><arg>${workingDir}/communityMap</arg>
|
||||||
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
<arg>--nameNode</arg><arg>${nameNode}</arg>
|
||||||
<arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
|
|
||||||
</java>
|
</java>
|
||||||
<ok to="get_ds_master_duplicate"/>
|
<ok to="get_ds_master_duplicate"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -164,32 +163,9 @@
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/publication</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/publicationextendedaffiliation</arg>
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
</spark>
|
<arg>--resultType</arg><arg>publication</arg>
|
||||||
<ok to="wait_eosc_dump"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
<action name="extend_publication_with_indicators">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>Extend Dump Publication with indicators </name>
|
|
||||||
<class>eu.dnetlib.dhp.oa.graph.dump.eosc.ExtendWithUsageCounts</class>
|
|
||||||
<jar>dump-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--actionSetPath</arg><arg>${actionSetPath}</arg>
|
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/publicationextendedaffiliation</arg>
|
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/publicationextended</arg>
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="wait_eosc_dump"/>
|
<ok to="wait_eosc_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -240,37 +216,13 @@
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/dataset</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/datasetextendedaffiliation</arg>
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--resultType</arg><arg>dataset</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="wait_eosc_dump"/>
|
<ok to="wait_eosc_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
</action>
|
</action>
|
||||||
<action name="extend_dataset_with_indicators">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>Extend Dump Dataset with indicators </name>
|
|
||||||
<class>eu.dnetlib.dhp.oa.graph.dump.eosc.ExtendWithUsageCounts</class>
|
|
||||||
<jar>dump-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--actionSetPath</arg><arg>${actionSetPath}</arg>
|
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/datasetextendedaffiliation</arg>
|
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/datasetextended</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="wait_eosc_dump"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
|
|
||||||
<action name="dump_eosc_orp">
|
<action name="dump_eosc_orp">
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||||
|
@ -316,32 +268,9 @@
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/otherresearchproduct</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/otherresearchproductextendedaffiliation</arg>
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
</spark>
|
<arg>--resultType</arg><arg>otherresearchproduct</arg>
|
||||||
<ok to="wait_eosc_dump"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
<action name="extend_orp_with_indicators">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>Extend Dump ORP with indicators </name>
|
|
||||||
<class>eu.dnetlib.dhp.oa.graph.dump.eosc.ExtendWithUsageCounts</class>
|
|
||||||
<jar>dump-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--actionSetPath</arg><arg>${actionSetPath}</arg>
|
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/otherresearchproductextendedaffiliation</arg>
|
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/otherresearchproductextended</arg>
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="wait_eosc_dump"/>
|
<ok to="wait_eosc_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -392,37 +321,15 @@
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/software</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/softwareextendedaffiliation</arg>
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
</spark>
|
<arg>--resultType</arg><arg>software</arg>
|
||||||
<ok to="wait_eosc_dump"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
<action name="extend_software_with_indicators">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>Extend Dump ORP with indicators </name>
|
|
||||||
<class>eu.dnetlib.dhp.oa.graph.dump.eosc.ExtendWithUsageCounts</class>
|
|
||||||
<jar>dump-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--actionSetPath</arg><arg>${actionSetPath}</arg>
|
|
||||||
<arg>--resultPath</arg><arg>${workingDir}/dump/softwareextendedaffiliation</arg>
|
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/softwareextended</arg>
|
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="wait_eosc_dump"/>
|
<ok to="wait_eosc_dump"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
</action>
|
</action>
|
||||||
|
|
||||||
|
|
||||||
<join name="wait_eosc_dump" to="prepareResultProject"/>
|
<join name="wait_eosc_dump" to="prepareResultProject"/>
|
||||||
|
|
||||||
<action name="prepareResultProject">
|
<action name="prepareResultProject">
|
||||||
|
@ -473,10 +380,12 @@
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${workingDir}/dump/publicationextendedaffiliation</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/publicationextendedproject</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
||||||
<arg>--dumpType</arg><arg>eosc</arg>
|
<arg>--dumpType</arg><arg>eosc</arg>
|
||||||
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--resultType</arg><arg>publication</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extend"/>
|
<ok to="join_extend"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -499,10 +408,12 @@
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${workingDir}/dump/datasetextendedaffiliation</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/datasetextendedproject</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
||||||
<arg>--dumpType</arg><arg>eosc</arg>
|
<arg>--dumpType</arg><arg>eosc</arg>
|
||||||
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--resultType</arg><arg>dataset</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extend"/>
|
<ok to="join_extend"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -525,10 +436,12 @@
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${workingDir}/dump/otherresearchproductextendedaffiliation</arg>
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/otherresearchproductextendedproject</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
||||||
<arg>--dumpType</arg><arg>eosc</arg>
|
<arg>--dumpType</arg><arg>eosc</arg>
|
||||||
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--resultType</arg><arg>otherresearchproduct</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extend"/>
|
<ok to="join_extend"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -551,11 +464,12 @@
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
</spark-opts>
|
</spark-opts>
|
||||||
<arg>--sourcePath</arg><arg>${workingDir}/dump/softwareextendedaffiliation
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
</arg>
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
<arg>--outputPath</arg><arg>${workingDir}/dump/softwareextendedproject</arg>
|
|
||||||
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
|
||||||
<arg>--dumpType</arg><arg>eosc</arg>
|
<arg>--dumpType</arg><arg>eosc</arg>
|
||||||
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--resultType</arg><arg>software</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_extend"/>
|
<ok to="join_extend"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
@ -696,7 +610,32 @@
|
||||||
</action>
|
</action>
|
||||||
|
|
||||||
|
|
||||||
<join name="join_extend_relation" to="make_archive"/>
|
<join name="join_extend_relation" to="dump_organization_project_relations"/>
|
||||||
|
|
||||||
|
<action name="dump_organization_project_relations">
|
||||||
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||||
|
<master>yarn</master>
|
||||||
|
<mode>cluster</mode>
|
||||||
|
<name>Dump for the relations between organization and projects in the subset of entities relevant for EOSC</name>
|
||||||
|
<class>eu.dnetlib.dhp.oa.graph.dump.eosc.SparkDumpOrganizationProject</class>
|
||||||
|
<jar>dump-${projectVersion}.jar</jar>
|
||||||
|
<spark-opts>
|
||||||
|
--executor-memory=${sparkExecutorMemory}
|
||||||
|
--executor-cores=${sparkExecutorCores}
|
||||||
|
--driver-memory=${sparkDriverMemory}
|
||||||
|
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||||
|
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||||
|
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||||
|
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||||
|
--conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir}
|
||||||
|
</spark-opts>
|
||||||
|
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||||
|
<arg>--outputPath</arg><arg>${outputPath}/dump/</arg>
|
||||||
|
<arg>--workingPath</arg><arg>${workingDir}/dump/</arg>
|
||||||
|
</spark>
|
||||||
|
<ok to="make_archive"/>
|
||||||
|
<error to="Kill"/>
|
||||||
|
</action>
|
||||||
<action name="make_archive">
|
<action name="make_archive">
|
||||||
<java>
|
<java>
|
||||||
<main-class>eu.dnetlib.dhp.oa.graph.dump.MakeTar</main-class>
|
<main-class>eu.dnetlib.dhp.oa.graph.dump.MakeTar</main-class>
|
||||||
|
|
|
@ -1,12 +1,7 @@
|
||||||
|
|
||||||
[
|
[
|
||||||
|
|
||||||
{
|
|
||||||
"paramName":"is",
|
|
||||||
"paramLongName":"isLookUpUrl",
|
|
||||||
"paramDescription": "URL of the isLookUp Service",
|
|
||||||
"paramRequired": true
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"paramName":"nn",
|
"paramName":"nn",
|
||||||
"paramLongName":"nameNode",
|
"paramLongName":"nameNode",
|
||||||
|
|
|
@ -28,7 +28,12 @@
|
||||||
"paramLongName":"workingPath",
|
"paramLongName":"workingPath",
|
||||||
"paramDescription": "The path to the community map",
|
"paramDescription": "The path to the community map",
|
||||||
"paramRequired": false
|
"paramRequired": false
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"paramName":"rt",
|
||||||
|
"paramLongName":"resultType",
|
||||||
|
"paramDescription": "The path to the community map",
|
||||||
|
"paramRequired": false}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
|
||||||
|
[
|
||||||
|
|
||||||
|
|
||||||
|
{
|
||||||
|
"paramName":"s",
|
||||||
|
"paramLongName":"sourcePath",
|
||||||
|
"paramDescription": "the name node",
|
||||||
|
"paramRequired": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "out",
|
||||||
|
"paramLongName": "outputPath",
|
||||||
|
"paramDescription": "the path used to store temporary output files",
|
||||||
|
"paramRequired": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "issm",
|
||||||
|
"paramLongName": "isSparkSessionManaged",
|
||||||
|
"paramDescription": "the path used to store temporary output files",
|
||||||
|
"paramRequired": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "wp",
|
||||||
|
"paramLongName": "workingPath",
|
||||||
|
"paramDescription": "the path used to store temporary output files",
|
||||||
|
"paramRequired": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -29,6 +29,18 @@
|
||||||
"paramLongName": "dumpType",
|
"paramLongName": "dumpType",
|
||||||
"paramDescription": "the dump type",
|
"paramDescription": "the dump type",
|
||||||
"paramRequired": false
|
"paramRequired": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "wp",
|
||||||
|
"paramLongName": "workingPath",
|
||||||
|
"paramDescription": "the working path",
|
||||||
|
"paramRequired": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "rt",
|
||||||
|
"paramLongName": "resultType",
|
||||||
|
"paramDescription": "the working path",
|
||||||
|
"paramRequired": false
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
[
|
||||||
|
|
||||||
|
{
|
||||||
|
"paramName":"s",
|
||||||
|
"paramLongName":"sourcePath",
|
||||||
|
"paramDescription": "the path of the sequencial file to read",
|
||||||
|
"paramRequired": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "out",
|
||||||
|
"paramLongName": "outputPath",
|
||||||
|
"paramDescription": "the path used to store temporary output files",
|
||||||
|
"paramRequired": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"paramName": "ssm",
|
||||||
|
"paramLongName": "isSparkSessionManaged",
|
||||||
|
"paramDescription": "true if the spark session is managed, false otherwise",
|
||||||
|
"paramRequired": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.eosc.model.Project;
|
import eu.dnetlib.dhp.eosc.model.ProjectSummary;
|
||||||
import eu.dnetlib.dhp.eosc.model.Result;
|
import eu.dnetlib.dhp.eosc.model.Result;
|
||||||
import eu.dnetlib.dhp.oa.graph.dump.eosc.SparkUpdateProjectInfo;
|
import eu.dnetlib.dhp.oa.graph.dump.eosc.SparkUpdateProjectInfo;
|
||||||
|
|
||||||
|
@ -194,15 +194,15 @@ public class UpdateProjectInfoTest {
|
||||||
.filter("id = '50|pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2' and code = '119027'")
|
.filter("id = '50|pensoft_____::00ea4a1cd53806a97d62ea6bf268f2a2' and code = '119027'")
|
||||||
.count());
|
.count());
|
||||||
|
|
||||||
Project project = verificationDataset
|
ProjectSummary project = verificationDataset
|
||||||
.map(
|
.map(
|
||||||
(MapFunction<Result, Project>) cr -> cr
|
(MapFunction<Result, ProjectSummary>) cr -> cr
|
||||||
.getProjects()
|
.getProjects()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(p -> p.getValidated() != null)
|
.filter(p -> p.getValidated() != null)
|
||||||
.collect(Collectors.toList())
|
.collect(Collectors.toList())
|
||||||
.get(0),
|
.get(0),
|
||||||
Encoders.bean(Project.class))
|
Encoders.bean(ProjectSummary.class))
|
||||||
.first();
|
.first();
|
||||||
|
|
||||||
Assertions.assertTrue(project.getFunder().getName().equals("Academy of Finland"));
|
Assertions.assertTrue(project.getFunder().getName().equals("Academy of Finland"));
|
||||||
|
@ -213,13 +213,13 @@ public class UpdateProjectInfoTest {
|
||||||
|
|
||||||
project = verificationDataset
|
project = verificationDataset
|
||||||
.map(
|
.map(
|
||||||
(MapFunction<Result, Project>) cr -> cr
|
(MapFunction<Result, ProjectSummary>) cr -> cr
|
||||||
.getProjects()
|
.getProjects()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(p -> p.getValidated() == null)
|
.filter(p -> p.getValidated() == null)
|
||||||
.collect(Collectors.toList())
|
.collect(Collectors.toList())
|
||||||
.get(0),
|
.get(0),
|
||||||
Encoders.bean(Project.class))
|
Encoders.bean(ProjectSummary.class))
|
||||||
.first();
|
.first();
|
||||||
|
|
||||||
Assertions.assertTrue(project.getFunder().getName().equals("European Commission"));
|
Assertions.assertTrue(project.getFunder().getName().equals("European Commission"));
|
||||||
|
|
|
@ -0,0 +1,205 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
import org.junit.jupiter.api.AfterAll;
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Affiliation;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Organization;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Relation;
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Result;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
public class ExtendAffiliationTest {
|
||||||
|
|
||||||
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
|
private static SparkSession spark;
|
||||||
|
|
||||||
|
private static Path workingDir;
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory
|
||||||
|
.getLogger(ExtendAffiliationTest.class);
|
||||||
|
|
||||||
|
private static HashMap<String, String> map = new HashMap<>();
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
public static void beforeAll() throws IOException {
|
||||||
|
workingDir = Files
|
||||||
|
.createTempDirectory(ExtendAffiliationTest.class.getSimpleName());
|
||||||
|
log.info("using work dir {}", workingDir);
|
||||||
|
|
||||||
|
SparkConf conf = new SparkConf();
|
||||||
|
conf.setAppName(ExtendAffiliationTest.class.getSimpleName());
|
||||||
|
|
||||||
|
conf.setMaster("local[*]");
|
||||||
|
conf.set("spark.driver.host", "localhost");
|
||||||
|
conf.set("hive.metastore.local", "true");
|
||||||
|
conf.set("spark.ui.enabled", "false");
|
||||||
|
conf.set("spark.sql.warehouse.dir", workingDir.toString());
|
||||||
|
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
|
||||||
|
|
||||||
|
spark = SparkSession
|
||||||
|
.builder()
|
||||||
|
.appName(ExtendAffiliationTest.class.getSimpleName())
|
||||||
|
.config(conf)
|
||||||
|
.getOrCreate();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterAll
|
||||||
|
public static void afterAll() throws IOException {
|
||||||
|
FileUtils.deleteDirectory(workingDir.toFile());
|
||||||
|
spark.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void ExtendEoscResultWithOrganizationTest() throws Exception {
|
||||||
|
final String sourcePath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final String workingPath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
|
||||||
|
spark
|
||||||
|
.read()
|
||||||
|
.textFile(workingPath + "publication")
|
||||||
|
.write()
|
||||||
|
.text(workingDir.toString() + "/working/publication");
|
||||||
|
|
||||||
|
ExtendEoscResultWithOrganizationStep2.main(new String[] {
|
||||||
|
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||||
|
"-outputPath", workingDir.toString() + "/",
|
||||||
|
"-sourcePath", sourcePath,
|
||||||
|
"-resultType", "publication",
|
||||||
|
"-workingPath", workingDir.toString() + "/working/"
|
||||||
|
});
|
||||||
|
|
||||||
|
/*
|
||||||
|
* affiliation relationships 20|13811704aa70::51a6ade52065e3b371d1ae822e07f1ff ->
|
||||||
|
* 50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba
|
||||||
|
*/
|
||||||
|
|
||||||
|
JavaRDD<Result> tmp = sc
|
||||||
|
.textFile(workingDir.toString() + "/working/publicationextendedaffiliation")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
|
||||||
|
|
||||||
|
Assertions.assertEquals(3, tmp.count());
|
||||||
|
|
||||||
|
tmp.foreach(p -> System.out.println(OBJECT_MAPPER.writeValueAsString(p)));
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
tmp
|
||||||
|
.filter(r -> Optional.ofNullable(r.getAffiliation()).isPresent() && r.getAffiliation().size() > 0)
|
||||||
|
.count());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
tmp
|
||||||
|
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
||||||
|
.first()
|
||||||
|
.getAffiliation()
|
||||||
|
.size());
|
||||||
|
|
||||||
|
List<Affiliation> affiliations = tmp
|
||||||
|
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
||||||
|
.first()
|
||||||
|
.getAffiliation();
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertTrue(
|
||||||
|
affiliations.stream().anyMatch(a -> a.getName().equalsIgnoreCase("Doris Engineering (France)")));
|
||||||
|
|
||||||
|
Affiliation organization = affiliations
|
||||||
|
.stream()
|
||||||
|
.filter(a -> a.getId().equalsIgnoreCase("20|13811704aa70::51a6ade52065e3b371d1ae822e07f1ff"))
|
||||||
|
.findFirst()
|
||||||
|
.get();
|
||||||
|
Assertions.assertEquals("Doris Engineering (France)", organization.getName());
|
||||||
|
Assertions
|
||||||
|
.assertTrue(
|
||||||
|
organization
|
||||||
|
.getPid()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(
|
||||||
|
p -> p.getValue().equalsIgnoreCase("grid.432986.2") && p.getType().equalsIgnoreCase("grid")));
|
||||||
|
Assertions
|
||||||
|
.assertTrue(
|
||||||
|
organization
|
||||||
|
.getPid()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(
|
||||||
|
p -> p.getValue().equalsIgnoreCase("https://ror.org/03nd0ms94")
|
||||||
|
&& p.getType().equalsIgnoreCase("ror")));
|
||||||
|
Assertions.assertEquals(2, organization.getPid().size());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void selectEoscResults() throws Exception {
|
||||||
|
|
||||||
|
final String sourcePath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final String workingPath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
ExtendEoscResultWithOrganizationStep2.main(new String[] {
|
||||||
|
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||||
|
"-outputPath", workingDir.toString() + "/",
|
||||||
|
"-sourcePath", sourcePath,
|
||||||
|
"-resultType", "publication",
|
||||||
|
"-workingPath", workingPath
|
||||||
|
});
|
||||||
|
|
||||||
|
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
|
||||||
|
JavaRDD<Organization> tmp = sc
|
||||||
|
.textFile(workingDir.toString() + "/organization")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Organization.class));
|
||||||
|
|
||||||
|
JavaRDD<Relation> rels = sc
|
||||||
|
.textFile(workingDir.toString() + "/resultOrganization")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
||||||
|
|
||||||
|
System.out.println(tmp.count());
|
||||||
|
|
||||||
|
Assertions.assertEquals(2, tmp.count());
|
||||||
|
|
||||||
|
Assertions.assertEquals(2, rels.count());
|
||||||
|
|
||||||
|
rels.foreach(r -> Assertions.assertTrue(r.getSource().startsWith("50|")));
|
||||||
|
rels.foreach(r -> Assertions.assertTrue(r.getTarget().startsWith("20|")));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,163 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
import org.junit.jupiter.api.AfterAll;
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.eosc.model.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
public class ExtendProjectTest {
|
||||||
|
|
||||||
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
|
private static SparkSession spark;
|
||||||
|
|
||||||
|
private static Path workingDir;
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory
|
||||||
|
.getLogger(ExtendProjectTest.class);
|
||||||
|
|
||||||
|
private static HashMap<String, String> map = new HashMap<>();
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
public static void beforeAll() throws IOException {
|
||||||
|
workingDir = Files
|
||||||
|
.createTempDirectory(ExtendProjectTest.class.getSimpleName());
|
||||||
|
log.info("using work dir {}", workingDir);
|
||||||
|
|
||||||
|
SparkConf conf = new SparkConf();
|
||||||
|
conf.setAppName(ExtendProjectTest.class.getSimpleName());
|
||||||
|
|
||||||
|
conf.setMaster("local[*]");
|
||||||
|
conf.set("spark.driver.host", "localhost");
|
||||||
|
conf.set("hive.metastore.local", "true");
|
||||||
|
conf.set("spark.ui.enabled", "false");
|
||||||
|
conf.set("spark.sql.warehouse.dir", workingDir.toString());
|
||||||
|
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
|
||||||
|
|
||||||
|
spark = SparkSession
|
||||||
|
.builder()
|
||||||
|
.appName(ExtendProjectTest.class.getSimpleName())
|
||||||
|
.config(conf)
|
||||||
|
.getOrCreate();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterAll
|
||||||
|
public static void afterAll() throws IOException {
|
||||||
|
FileUtils.deleteDirectory(workingDir.toFile());
|
||||||
|
spark.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void ExtendEoscResultWithProjectTest() throws Exception {
|
||||||
|
final String sourcePath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final String workingPath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
|
||||||
|
spark
|
||||||
|
.read()
|
||||||
|
.textFile(workingPath + "publication")
|
||||||
|
.write()
|
||||||
|
.text(workingDir.toString() + "/working/publicationextendedaffiliation");
|
||||||
|
|
||||||
|
SparkUpdateProjectInfo.main(new String[] {
|
||||||
|
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||||
|
"-outputPath", workingDir.toString() + "/",
|
||||||
|
"-sourcePath", sourcePath,
|
||||||
|
"-resultType", "publication",
|
||||||
|
"-workingPath", workingDir.toString() + "/working/",
|
||||||
|
"-preparedInfoPath", workingPath + "preparedInfo"
|
||||||
|
});
|
||||||
|
|
||||||
|
JavaRDD<Result> tmp = sc
|
||||||
|
.textFile(workingDir.toString() + "/working/publicationextendedproject")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
|
||||||
|
|
||||||
|
Assertions.assertEquals(3, tmp.count());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
2,
|
||||||
|
tmp
|
||||||
|
.filter(r -> Optional.ofNullable(r.getProjects()).isPresent() && r.getProjects().size() > 0)
|
||||||
|
.count());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
2,
|
||||||
|
tmp
|
||||||
|
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
||||||
|
.first()
|
||||||
|
.getProjects()
|
||||||
|
.size());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
3,
|
||||||
|
tmp
|
||||||
|
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
||||||
|
.first()
|
||||||
|
.getProjects()
|
||||||
|
.size());
|
||||||
|
|
||||||
|
List<ProjectSummary> projectSummaries = tmp
|
||||||
|
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
||||||
|
.first()
|
||||||
|
.getProjects();
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertTrue(
|
||||||
|
projectSummaries.stream().anyMatch(p -> p.getFunder().getShortName().equals("NSF")));
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertTrue(
|
||||||
|
projectSummaries.stream().anyMatch(p -> p.getFunder().getShortName().equals("UKRI")));
|
||||||
|
|
||||||
|
JavaRDD<Project> projects = sc
|
||||||
|
.textFile(workingDir.toString() + "/project")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Project.class));
|
||||||
|
|
||||||
|
JavaRDD<Relation> rels = sc
|
||||||
|
.textFile(workingDir.toString() + "/resultProject")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
||||||
|
|
||||||
|
System.out.println(projects.count());
|
||||||
|
|
||||||
|
Assertions.assertEquals(5, projects.count());
|
||||||
|
|
||||||
|
Assertions.assertEquals(5, rels.count());
|
||||||
|
|
||||||
|
rels.foreach(r -> Assertions.assertTrue(r.getSource().startsWith("50|")));
|
||||||
|
rels.foreach(r -> Assertions.assertTrue(r.getTarget().startsWith("40|")));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,132 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.graph.dump.eosc;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.spark.SparkConf;
|
||||||
|
import org.apache.spark.api.java.JavaRDD;
|
||||||
|
import org.apache.spark.api.java.JavaSparkContext;
|
||||||
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
import org.junit.jupiter.api.AfterAll;
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Relation;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author miriam.baglioni
|
||||||
|
* @Date 25/10/23
|
||||||
|
*/
|
||||||
|
public class OrganizationProjectRelationTest {
|
||||||
|
|
||||||
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
|
private static SparkSession spark;
|
||||||
|
|
||||||
|
private static Path workingDir;
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory
|
||||||
|
.getLogger(OrganizationProjectRelationTest.class);
|
||||||
|
|
||||||
|
private static HashMap<String, String> map = new HashMap<>();
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
public static void beforeAll() throws IOException {
|
||||||
|
workingDir = Files
|
||||||
|
.createTempDirectory(OrganizationProjectRelationTest.class.getSimpleName());
|
||||||
|
log.info("using work dir {}", workingDir);
|
||||||
|
|
||||||
|
SparkConf conf = new SparkConf();
|
||||||
|
conf.setAppName(OrganizationProjectRelationTest.class.getSimpleName());
|
||||||
|
|
||||||
|
conf.setMaster("local[*]");
|
||||||
|
conf.set("spark.driver.host", "localhost");
|
||||||
|
conf.set("hive.metastore.local", "true");
|
||||||
|
conf.set("spark.ui.enabled", "false");
|
||||||
|
conf.set("spark.sql.warehouse.dir", workingDir.toString());
|
||||||
|
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
|
||||||
|
|
||||||
|
spark = SparkSession
|
||||||
|
.builder()
|
||||||
|
.appName(OrganizationProjectRelationTest.class.getSimpleName())
|
||||||
|
.config(conf)
|
||||||
|
.getOrCreate();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterAll
|
||||||
|
public static void afterAll() throws IOException {
|
||||||
|
FileUtils.deleteDirectory(workingDir.toFile());
|
||||||
|
spark.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void ExtendEoscResultWithProjectTest() throws Exception {
|
||||||
|
final String sourcePath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final String workingPath = getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/")
|
||||||
|
.getPath();
|
||||||
|
|
||||||
|
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
||||||
|
|
||||||
|
spark
|
||||||
|
.read()
|
||||||
|
.textFile(workingPath + "organization")
|
||||||
|
.write()
|
||||||
|
.text(workingDir.toString() + "/working/organization");
|
||||||
|
|
||||||
|
spark
|
||||||
|
.read()
|
||||||
|
.textFile(workingPath + "project")
|
||||||
|
.write()
|
||||||
|
.text(workingDir.toString() + "/working/project");
|
||||||
|
|
||||||
|
SparkDumpOrganizationProject.main(new String[] {
|
||||||
|
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
||||||
|
"-outputPath", workingDir.toString() + "/working/",
|
||||||
|
"-sourcePath", sourcePath
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
JavaRDD<Relation> tmp = sc
|
||||||
|
.textFile(workingDir.toString() + "/working/organizationProject")
|
||||||
|
.map(item -> OBJECT_MAPPER.readValue(item, Relation.class));
|
||||||
|
|
||||||
|
Assertions.assertEquals(3, tmp.count());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
tmp
|
||||||
|
.filter(r -> r.getSource().equalsIgnoreCase("20|chistera____::9146e9ef10640675f361d674e77bd254"))
|
||||||
|
.count());
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
2,
|
||||||
|
tmp
|
||||||
|
.filter(r -> r.getSource().equalsIgnoreCase("20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64"))
|
||||||
|
.count());
|
||||||
|
|
||||||
|
Assertions
|
||||||
|
.assertEquals(
|
||||||
|
1,
|
||||||
|
tmp
|
||||||
|
.filter(
|
||||||
|
r -> r.getSource().equalsIgnoreCase("20|chistera____::9146e9ef10640675f361d674e77bd254") &&
|
||||||
|
r.getTarget().equalsIgnoreCase("40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621"))
|
||||||
|
.count());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -24,8 +24,8 @@ import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.eosc.model.Affiliation;
|
||||||
import eu.dnetlib.dhp.eosc.model.Indicator;
|
import eu.dnetlib.dhp.eosc.model.Indicator;
|
||||||
import eu.dnetlib.dhp.eosc.model.Organization;
|
|
||||||
import eu.dnetlib.dhp.eosc.model.Result;
|
import eu.dnetlib.dhp.eosc.model.Result;
|
||||||
import eu.dnetlib.dhp.schema.action.AtomicAction;
|
import eu.dnetlib.dhp.schema.action.AtomicAction;
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
@ -128,136 +128,6 @@ public class SelectEoscResultTest {
|
||||||
// legalname = MIKARE RESEARCH
|
// legalname = MIKARE RESEARCH
|
||||||
// pid = []
|
// pid = []
|
||||||
// for 50|06cdd3ff4700::ff21e3c55d527fa7db171137c5fd1f1f no affiliation relation is provided
|
// for 50|06cdd3ff4700::ff21e3c55d527fa7db171137c5fd1f1f no affiliation relation is provided
|
||||||
@Test
|
|
||||||
public void ExtendEoscResultWithOrganizationTest() throws Exception {
|
|
||||||
final String sourcePath = getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/input")
|
|
||||||
.getPath();
|
|
||||||
|
|
||||||
final String cmp = getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/communityMapPath/communitymap.json")
|
|
||||||
.getPath();
|
|
||||||
|
|
||||||
String resultPath = getClass()
|
|
||||||
.getResource("/eu/dnetlib/dhp/oa/graph/dump/eosc/working/publication")
|
|
||||||
.getPath();
|
|
||||||
|
|
||||||
ExtendEoscResultWithOrganizationStep2.main(new String[] {
|
|
||||||
"-isSparkSessionManaged", Boolean.FALSE.toString(),
|
|
||||||
"-outputPath", workingDir.toString() + "/publication",
|
|
||||||
"-sourcePath", sourcePath,
|
|
||||||
// "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication",
|
|
||||||
"-resultPath", resultPath
|
|
||||||
});
|
|
||||||
|
|
||||||
final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext());
|
|
||||||
|
|
||||||
JavaRDD<Result> tmp = sc
|
|
||||||
.textFile(workingDir.toString() + "/publication")
|
|
||||||
.map(item -> OBJECT_MAPPER.readValue(item, Result.class));
|
|
||||||
|
|
||||||
Assertions.assertEquals(3, tmp.count());
|
|
||||||
|
|
||||||
Assertions
|
|
||||||
.assertEquals(
|
|
||||||
2,
|
|
||||||
tmp
|
|
||||||
.filter(r -> Optional.ofNullable(r.getAffiliation()).isPresent() && r.getAffiliation().size() > 0)
|
|
||||||
.count());
|
|
||||||
|
|
||||||
Assertions
|
|
||||||
.assertEquals(
|
|
||||||
2,
|
|
||||||
tmp
|
|
||||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation()
|
|
||||||
.size());
|
|
||||||
|
|
||||||
List<Organization> affiliations = tmp
|
|
||||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation();
|
|
||||||
|
|
||||||
Assertions
|
|
||||||
.assertTrue(
|
|
||||||
affiliations.stream().anyMatch(a -> a.getName().equalsIgnoreCase("Doris Engineering (France)")));
|
|
||||||
Assertions.assertTrue(affiliations.stream().anyMatch(a -> a.getName().equalsIgnoreCase("RENNES METROPOLE")));
|
|
||||||
|
|
||||||
Organization organization = affiliations
|
|
||||||
.stream()
|
|
||||||
.filter(a -> a.getId().equalsIgnoreCase("20|13811704aa70::51a6ade52065e3b371d1ae822e07f1ff"))
|
|
||||||
.findFirst()
|
|
||||||
.get();
|
|
||||||
Assertions.assertEquals("Doris Engineering (France)", organization.getName());
|
|
||||||
Assertions
|
|
||||||
.assertTrue(
|
|
||||||
organization
|
|
||||||
.getPid()
|
|
||||||
.stream()
|
|
||||||
.anyMatch(
|
|
||||||
p -> p.getValue().equalsIgnoreCase("grid.432986.2") && p.getType().equalsIgnoreCase("grid")));
|
|
||||||
Assertions
|
|
||||||
.assertTrue(
|
|
||||||
organization
|
|
||||||
.getPid()
|
|
||||||
.stream()
|
|
||||||
.anyMatch(
|
|
||||||
p -> p.getValue().equalsIgnoreCase("https://ror.org/03nd0ms94")
|
|
||||||
&& p.getType().equalsIgnoreCase("ror")));
|
|
||||||
Assertions.assertEquals(2, organization.getPid().size());
|
|
||||||
|
|
||||||
organization = affiliations
|
|
||||||
.stream()
|
|
||||||
.filter(a -> a.getId().equalsIgnoreCase("20|MetisRadboud::b58bdbe8ae5acead04fc76777d2f8017"))
|
|
||||||
.findFirst()
|
|
||||||
.get();
|
|
||||||
Assertions.assertEquals("RENNES METROPOLE", organization.getName());
|
|
||||||
Assertions.assertEquals(1, organization.getPid().size());
|
|
||||||
Assertions
|
|
||||||
.assertTrue(
|
|
||||||
organization.getPid().get(0).getValue().equalsIgnoreCase("892062829")
|
|
||||||
&& organization.getPid().get(0).getType().equalsIgnoreCase("pic"));
|
|
||||||
|
|
||||||
Assertions
|
|
||||||
.assertEquals(
|
|
||||||
1,
|
|
||||||
tmp
|
|
||||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation()
|
|
||||||
.size());
|
|
||||||
Assertions
|
|
||||||
.assertEquals(
|
|
||||||
"MIKARE RESEARCH",
|
|
||||||
tmp
|
|
||||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation()
|
|
||||||
.get(0)
|
|
||||||
.getName());
|
|
||||||
Assertions
|
|
||||||
.assertEquals(
|
|
||||||
0,
|
|
||||||
tmp
|
|
||||||
.filter(r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation()
|
|
||||||
.get(0)
|
|
||||||
.getPid()
|
|
||||||
.size());
|
|
||||||
|
|
||||||
Assertions
|
|
||||||
.assertFalse(
|
|
||||||
Optional
|
|
||||||
.ofNullable(
|
|
||||||
tmp
|
|
||||||
.filter(
|
|
||||||
r -> r.getId().equalsIgnoreCase("50|06cdd3ff4700::ff21e3c55d527fa7db171137c5fd1f1f"))
|
|
||||||
.first()
|
|
||||||
.getAffiliation())
|
|
||||||
.isPresent());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void verifyIndicatorsTest() throws Exception {
|
public void verifyIndicatorsTest() throws Exception {
|
||||||
|
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -9,4 +9,12 @@
|
||||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|____________::d1b0ee22411434cf905692d0fac25749","subRelType":"affiliation","target":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|____________::d1b0ee22411434cf905692d0fac25749","subRelType":"affiliation","target":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|____________::d1b0ee22411434cf905692d0fac25749","subRelType":"affiliation","target":"50|pmid________::3a5bb2b50c18e755cbe67b9ca7d821ee","validated":false}
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|____________::d1b0ee22411434cf905692d0fac25749","subRelType":"affiliation","target":"50|pmid________::3a5bb2b50c18e755cbe67b9ca7d821ee","validated":false}
|
||||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|aka_________::04ab269cfcf6bd571b6285151ec554b5","subRelType":"affiliation","target":"50|nora_uio__no::01152f3e683765695bbad68fc692b85e","validated":false}
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|aka_________::04ab269cfcf6bd571b6285151ec554b5","subRelType":"affiliation","target":"50|nora_uio__no::01152f3e683765695bbad68fc692b85e","validated":false}
|
||||||
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|aka_________::0838366fa1df3c1599ddefc2168ada5d","subRelType":"affiliation","target":"50|arXiv_______::abe2b16af6067994dda4beab6410b35d","validated":false}
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","source":"20|aka_________::0838366fa1df3c1599ddefc2168ada5d","subRelType":"affiliation","target":"50|arXiv_______::abe2b16af6067994dda4beab6410b35d","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|chistera____::9146e9ef10640675f361d674e77bd254","subRelType":"affiliation","source":"50|355e65625b88::38d0ab3b2212878dee7072170f1561ee","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|chistera____::9146e9ef10640675f361d674e77bd254","subRelType":"affiliation","source":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|aka_________::04ab269cfcf6bd571b6285151ec554b5","subRelType":"affiliation","source":"50|355e65625b88::38d0ab3b2212878dee7072170f1561ee","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isAuthorInstitutionOf","relType":"resultOrganization","target":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","subRelType":"affiliation","source":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466741040,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","subRelType":"participation","source":"20|chistera____::9146e9ef10640675f361d674e77bd254","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8847"},"lastupdatetimestamp":1658466737372,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|ukri________::081b09db1211a7b89eb3610d3160e9ba","subRelType":"participation","source":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466717565,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","subRelType":"participation","source":"20|nih_________::7523ba08be91b521952082f0c25daf5f","validated":false}
|
||||||
|
{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"iis::document_affiliations","inferred":true,"invisible":false,"provenanceaction":{"classid":"iis","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.8998"},"lastupdatetimestamp":1658466733174,"properties":[],"relClass":"isParticipant","relType":"resultOrganization","target":"40|corda__h2020::5e49c0ee515f36e416a00cc292dfb310","subRelType":"participation","source":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","validated":false}
|
|
@ -0,0 +1,2 @@
|
||||||
|
{"legalshortname":null,"legalname":"School of Computer Science, University of Birmingham","websiteurl":null,"alternativenames":[],"country":{"code":"GB","label":"United Kingdom"},"id":"20|chistera____::9146e9ef10640675f361d674e77bd254","pid":[]}
|
||||||
|
{"legalshortname":"EVOTHINGS","legalname":"EVOTHINGS AB","websiteurl":"https://evothings.com","alternativenames":[],"country":{"code":"SE","label":"Sweden"},"id":"20|corda__h2020::dfe84ab5cad50d4dcfaf5bd0c86e1b64","pid":[{"type":"PIC","value":"922724335"}]}
|
|
@ -0,0 +1,2 @@
|
||||||
|
{"projectsList":[{"code":"0430175","funder":{"fundingStream":"Directorate for Computer & Information Science & Engineering","jurisdiction":"US","name":"National Science Foundation","shortName":"NSF"},"id":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","provenance":{"provenance":"Harvested","trust":"0.900"},"title":"Collaborative Research: Temporal Aspects"},{"code":"EP/F01161X/1","funder":{"fundingStream":"EPSRC","jurisdiction":"GB","name":"UK Research and Innovation","shortName":"UKRI"},"id":"40|ukri________::081b09db1211a7b89eb3610d3160e9ba","provenance":{"provenance":"Harvested","trust":"0.900"},"title":"The complexity of valued constraints"}],"resultId":"50|06cdd3ff4700::93859bd27121c3ee7c6ee4bfb1790cba"}
|
||||||
|
{"projectsList":[{"acronym":"METIS","code":"317669","funder":{"fundingStream":"FP7","jurisdiction":"EU","name":"European Commission","shortName":"EC"},"id":"40|corda_______::175629cbea2038ed02c85e7132fc4be2","provenance":{"provenance":"Harvested","trust":"0.900"},"title":"Mobile and wireless communications Enablers for Twenty-twenty (2020) Information Society"},{"code":"unidentified","funder":{"jurisdiction":"CA","name":"Natural Sciences and Engineering Research Council of Canada","shortName":"NSERC"},"id":"40|nserc_______::1e5e62235d094afd01cd56e65112fc63","provenance":{"provenance":"Harvested","trust":"0.900"},"title":"unidentified"},{"acronym":"MiLC","code":"753431","funder":{"fundingStream":"H2020","jurisdiction":"EU","name":"European Commission","shortName":"EC"},"id":"40|corda__h2020::5e49c0ee515f36e416a00cc292dfb310","provenance":{"provenance":"Harvested","trust":"0.900"},"title":"Monotonicity in Logic and Complexity"}],"resultId":"50|06cdd3ff4700::cd7711c65d518859f1d87056e2c45d98"}
|
|
@ -0,0 +1,5 @@
|
||||||
|
{"id":"40|corda__h2020::5e49c0ee515f36e416a00cc292dfb310","websiteurl":null,"code":"287027","acronym":null,"title":"Advanced numerical computation methods for massive parabolic problems","startdate":"2015-09-01","enddate":"2019-08-31","callidentifier":"Academy Project LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":644270.0},"h2020programme":[]}
|
||||||
|
{"id":"40|nsf_________::d1c070f4252c32e23ccc3f4211c9c621","websiteurl":null,"code":"328474","acronym":null,"title":"A novel family-based sequencing approach and dissection of regulatory networks underlying a colour polymorphism","startdate":"2020-01-01","enddate":"2022-01-31","callidentifier":"Molecular Regulatory Networks of Life (R’Life), call for Academy-funded researchers BTY","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":250000.0},"h2020programme":[]}
|
||||||
|
{"id":"40|nserc_______::1e5e62235d094afd01cd56e65112fc63","websiteurl":null,"code":"107987","acronym":null,"title":"Structure of the upper mantle beneath Central Fennoscandian Shield from seismic anisotropy studies","startdate":"2005-02-15","enddate":"2005-12-31","callidentifier":"Apurahat tutkijainvaihtoon ja muuhun kahdenvälisiin sopimuksiin perustuvaan yhteistoimintaan LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":700.0},"h2020programme":[]}
|
||||||
|
{"id":"40|corda_______::175629cbea2038ed02c85e7132fc4be2","websiteurl":null,"code":"201608","acronym":null,"title":"Symbiotic Legumes For Sustainable Food Production and Prevention of Land Deglaration in China","startdate":"2003-01-01","enddate":"2005-12-31","callidentifier":"Appropriations for development studies BY","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":300010.0},"h2020programme":[]}
|
||||||
|
{"id":"40|ukri________::081b09db1211a7b89eb3610d3160e9ba","websiteurl":null,"code":"200618","acronym":null,"title":"Atomic Emission.","startdate":"2002-04-24","enddate":"2002-12-31","callidentifier":"Researcher exchange to Finland LT","keywords":null,"openaccessmandateforpublications":false,"openaccessmandatefordataset":false,"subject":[],"funding":[{"shortName":"AKA","name":"Academy of Finland","jurisdiction":"FI","fundingStream":null,"funding_stream":null}],"summary":null,"granted":{"currency":"EUR","totalcost":0.0,"fundedamount":2760.0},"h2020programme":[]}
|
3
pom.xml
3
pom.xml
|
@ -6,6 +6,7 @@
|
||||||
<modules>
|
<modules>
|
||||||
<module>dump-schema</module>
|
<module>dump-schema</module>
|
||||||
<module>dump</module>
|
<module>dump</module>
|
||||||
|
<module>api</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<parent>
|
<parent>
|
||||||
|
@ -102,7 +103,7 @@
|
||||||
<junit-jupiter.version>5.6.1</junit-jupiter.version>
|
<junit-jupiter.version>5.6.1</junit-jupiter.version>
|
||||||
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
|
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
|
||||||
<dhp.guava.version>11.0.2</dhp.guava.version>
|
<dhp.guava.version>11.0.2</dhp.guava.version>
|
||||||
<dhp-schemas.version>[3.17.1]</dhp-schemas.version>
|
<dhp-schemas.version>[4.17.2]</dhp-schemas.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
</project>
|
</project>
|
Loading…
Reference in New Issue