D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

removed last islookupur. Added new data for the irish tender

This commit is contained in:
Miriam Baglioni 2023-12-19 11:08:04 +01:00
parent 14e2027bb0
commit ac8ef53d02
9 changed files with 156 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/OpenAccessColor.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.dhp.oa.model;
/**
* @author miriam.baglioni
* @Date 19/12/23
*/
/**
* The OpenAccess color meant to be used on the result level
*/
public enum OpenAccessColor {
gold, hybrid, bronze
}

48
dump-schema/src/main/java/eu/dnetlib/dhp/oa/model/Result.java
View File

@ -5,6 +5,7 @@ import java.io.Serializable;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.github.imifou.jsonschema.module.addon.annotation.JsonSchema;
/**
@ -74,6 +75,53 @@ public class Result implements Serializable {
private List<Author> author;
// resulttype allows subclassing results into publications | datasets | software
@JsonProperty("isGreen")
@JsonSchema(description = "True if the result is green Open Access")
private Boolean isGreen;
@JsonSchema(description = "The Open Access Color of the publication")
private OpenAccessColor openAccessColor;
@JsonProperty("isInDiamondJournal")
@JsonSchema(description = "True if the result is published in a Diamond Journal")
private Boolean isInDiamondJournal;
@JsonSchema(description = "True if the result is outcome of a project")
private Boolean publiclyFunded;
public Boolean getGreen() {
return isGreen;
}
public void setGreen(Boolean green) {
isGreen = green;
}
public OpenAccessColor getOpenAccessColor() {
return openAccessColor;
}
public void setOpenAccessColor(OpenAccessColor openAccessColor) {
this.openAccessColor = openAccessColor;
}
public Boolean getInDiamondJournal() {
return isInDiamondJournal;
}
public void setInDiamondJournal(Boolean inDiamondJournal) {
isInDiamondJournal = inDiamondJournal;
}
public Boolean getPubliclyFunded() {
return publiclyFunded;
}
public void setPubliclyFunded(Boolean publiclyFunded) {
this.publiclyFunded = publiclyFunded;
}
@JsonSchema(
description = "Type of the result: one of 'publication', 'dataset', 'software', 'other' (see also https://api.openaire.eu/vocabularies/dnet:result_typologies)")
private String type; // resulttype

9
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java
View File

@ -7,8 +7,8 @@ import java.io.Serializable;
import java.util.*;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.oa.model.OpenAccessColor;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.CardinalityTooHighException;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
@ -16,7 +16,6 @@ import eu.dnetlib.dhp.oa.model.*;
import eu.dnetlib.dhp.oa.model.AccessRight;
import eu.dnetlib.dhp.oa.model.Author;
import eu.dnetlib.dhp.oa.model.GeoLocation;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.oa.model.Instance;
import eu.dnetlib.dhp.oa.model.OpenAccessRoute;
import eu.dnetlib.dhp.oa.model.Result;
@ -54,6 +53,10 @@ public class ResultMapper implements Serializable {
mapCountry(out, input);
mapCoverage(out, input);
out.setDateofcollection(input.getDateofcollection());
out.setGreen(input.getIsGreen());
out.setInDiamondJournal(input.getIsInDiamondJournal());
out.setPubliclyFunded(input.getPubliclyFunded());
out.setOpenAccessColor(OpenAccessColor.valueOf(input.getOpenAccessColor().name()));
mapDescription(out, input);
mapEmbargo(out, input);
mapFormat(out, input);
@ -671,7 +674,7 @@ public class ResultMapper implements Serializable {
Provenance p = new Provenance();
p.setProvenance(di.get().getProvenanceaction().getClassname());
if (!s.getQualifier().getClassid().equalsIgnoreCase("fos") &&
s.getQualifier().getClassid().equalsIgnoreCase("sdg"))
!s.getQualifier().getClassid().equalsIgnoreCase("sdg"))
p.setTrust(di.get().getTrust());
subject.setProvenance(p);
}

2
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/SaveCommunityMap.java
View File

@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
/**
* This class connects with the IS related to the isLookUpUrl got as parameter. It saves the information about the
* This class connects with the community APIs for production. It saves the information about the
* context that will guide the dump of the results. The information saved is a HashMap. The key is the id of a community
* - research infrastructure/initiative , the value is the label of the research community - research
* infrastructure/initiative.

28
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Utils.java
View File

@ -22,9 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import eu.dnetlib.dhp.common.HdfsSupport;
import eu.dnetlib.dhp.communityapi.QueryCommunityAPI;
import eu.dnetlib.dhp.communityapi.model.CommunityModel;
import eu.dnetlib.dhp.communityapi.model.CommunitySummary;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.graph.dump.complete.Constants;
import eu.dnetlib.dhp.oa.model.Indicator;
@ -36,8 +34,7 @@ import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity;
import eu.dnetlib.dhp.schema.oaf.KeyValue;
import eu.dnetlib.dhp.schema.oaf.Measure;
import eu.dnetlib.dhp.utils.DHPUtils;
import eu.dnetlib.dhp.utils.ISLookupClientFactory;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import scala.Tuple2;
public class Utils {
@ -58,9 +55,6 @@ public class Utils {
.map((MapFunction<String, R>) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz));
}
public static ISLookUpService getIsLookUpService(String isLookUpUrl) {
return ISLookupClientFactory.getLookUpService(isLookUpUrl);
}
public static String getContextId(String id) {
@ -166,23 +160,7 @@ public class Utils {
default:
getImpactMeasure(i).add(getScore(m.getId(), m.getUnit()));
break;
// case IMPACT_POPULARITY:
// getImpactMeasure(i).setPopularity(getScore(m.getUnit()));
// break;
// case IMPACT_POPULARITY_ALT:
// getImpactMeasure(i).setPopularity_alt(getScore(m.getUnit()));
// break;
// case IMPACT_IMPULSE:
// getImpactMeasure(i).setImpulse(getScore(m.getUnit()));
// break;
// case IMPACT_INFLUENCE:
// getImpactMeasure(i).setInfluence(getScore(m.getUnit()));
// break;
// case IMPACT_INFLUENCE_ALT:
// getImpactMeasure(i).setInfluence_alt(getScore(m.getUnit()));
// break;
// default:
// break;
}
}

7
dump/src/main/resources/eu/dnetlib/dhp/oa/graph/dump/input_dump_csv_ste1.json
View File

@ -13,13 +13,6 @@
"paramDescription": "true if the spark session is managed, false otherwise",
"paramRequired": true
},
{
"paramName":"ilu",
"paramLongName":"isLookUpUrl",
"paramDescription": "the name of the result table we are currently working on",
"paramRequired": true
},
{
"paramName":"c",
"paramLongName":"communities",

5
dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/CreateEntityTest.java
View File

@ -34,8 +34,7 @@ import eu.dnetlib.dhp.oa.graph.dump.UtilCommunityAPI;
import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity;
import eu.dnetlib.dhp.oa.model.graph.ResearchInitiative;
import eu.dnetlib.dhp.utils.DHPUtils;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
@ExtendWith(MockitoExtension.class)
public class CreateEntityTest {
@ -118,7 +117,7 @@ public class CreateEntityTest {
@Test
@Disabled
void test2() throws IOException, ISLookUpException {
void test2() throws IOException {
LocalFileSystem fs = FileSystem.getLocal(new Configuration());
Path hdfsWritePath = new Path(workingDir + "/prova");

81
dump/wget-log Normal file
View File

@ -0,0 +1,81 @@
--2023-11-30 16:20:33-- http://10.5555/1071509.1071530
Resolving 10.5555 (10.5555)... 10.0.21.179
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 09:02:48-- (try: 2) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 11:03:58-- (try: 3) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:04:08-- (try: 4) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:37:35-- (try: 5) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:38:55-- (try: 6) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:40:16-- (try: 7) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:41:38-- (try: 8) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:43:01-- (try: 9) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:44:25-- (try:10) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 13:45:50-- (try:11) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:35:37-- (try:12) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:37:02-- (try:13) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:38:27-- (try:14) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:39:52-- (try:15) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:41:17-- (try:16) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:42:42-- (try:17) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:44:07-- (try:18) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:45:32-- (try:19) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Retrying.
--2023-12-11 14:46:57-- (try:20) http://10.5555/1071509.1071530
Connecting to 10.5555 (10.5555)|10.0.21.179|:80... failed: Operation timed out.
Giving up.

2
pom.xml
View File

@ -103,7 +103,7 @@
<junit-jupiter.version>5.6.1</junit-jupiter.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<dhp.guava.version>11.0.2</dhp.guava.version>
<dhp-schemas.version>[3.17.1]</dhp-schemas.version>
<dhp-schemas.version>[4.17.2]</dhp-schemas.version>
</properties>
</project>