miriam.baglioni
/
dnet-hadoop
forked from D-Net/dnet-hadoop
2
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

fix to the mapper, and changed of the json for testing

This commit is contained in:
Miriam Baglioni 2020-06-23 11:07:42 +02:00
parent 3da12be81f
commit 33e2ebeaaa
3 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Constants.java
View File

@ -11,6 +11,7 @@ public class Constants {
public static String DATASET_URL = "https://beta.risis.openaire.eu/search/dataset?datasetId="; public static String DATASET_URL = "https://beta.risis.openaire.eu/search/dataset?datasetId=";
public static String SOFTWARE_URL = "https://beta.risis.openaire.eu/search/software?softwareId="; public static String SOFTWARE_URL = "https://beta.risis.openaire.eu/search/software?softwareId=";
public static String ORP_URL = "https://beta.risis.openaire.eu/search/other?orpId="; public static String ORP_URL = "https://beta.risis.openaire.eu/search/other?orpId=";
public static String DEFAULT_LICENCE_ID = "notspecified";
public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap(); public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap(); public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
public static final Map<String, String> gcatCatalogue = Maps.newHashMap(); public static final Map<String, String> gcatCatalogue = Maps.newHashMap();
@ -34,11 +35,11 @@ public class Constants {
static { static {
gcatCatalogue.put("OPEN", "OPEN"); gcatCatalogue.put("OPEN", "OPEN");
accessRightsCoarMap.put("RESTRICTED", "RESTRICTED"); gcatCatalogue.put("RESTRICTED", "RESTRICTED");
accessRightsCoarMap.put("OPEN SOURCE", "OPEN"); gcatCatalogue.put("OPEN SOURCE", "OPEN");
accessRightsCoarMap.put("CLOSED", "CLOSED"); gcatCatalogue.put("CLOSED", "CLOSED");
accessRightsCoarMap.put("EMBARGO", "EMBARGO"); gcatCatalogue.put("EMBARGO", "EMBARGO");
accessRightsCoarMap.put("UNKNOWN", "UNKNOWN"); gcatCatalogue.put("UNKNOWN", "UNKNOWN");
accessRightsCoarMap.put("OTHER", "UNKNOWN"); gcatCatalogue.put("OTHER", "UNKNOWN");
} }
} }

16
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/gcat/Mapper.java
View File

@ -44,17 +44,13 @@ public class Mapper implements Serializable {
externals.add(KeyValue.newInstance("result type", "dataset")); externals.add(KeyValue.newInstance("result type", "dataset"));
break; break;
case "software": case "software":
eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input; eu.dnetlib.dhp.schema.oaf.Software is = (eu.dnetlib.dhp.schema.oaf.Software) input;
Optional Optional
.ofNullable(is.getCodeRepositoryUrl()) .ofNullable(is.getCodeRepositoryUrl())
.ifPresent(value -> urlSet.add(value.getValue())); .ifPresent(value -> urlSet.add(value.getValue()));
Optional Optional
.ofNullable(is.getDocumentationUrl()) .ofNullable(is.getDocumentationUrl())
.ifPresent( .ifPresent(value -> value.forEach(v -> urlSet.add(v.getValue())));
value -> value
.stream()
.map(v -> urlSet.add(v.getValue())));
Optional Optional
.ofNullable(is.getProgrammingLanguage()) .ofNullable(is.getProgrammingLanguage())
@ -71,6 +67,8 @@ public class Mapper implements Serializable {
} }
out.setLicence_id(Constants.DEFAULT_LICENCE_ID);
Optional Optional
.ofNullable(input.getAuthor()) .ofNullable(input.getAuthor())
.ifPresent( .ifPresent(
@ -106,8 +104,7 @@ public class Mapper implements Serializable {
.ofNullable(input.getDescription()) .ofNullable(input.getDescription())
.ifPresent(value -> .ifPresent(value ->
getDescription(out, externals, value) getDescription(out, externals, value));
);
Optional Optional
.ofNullable(input.getEmbargoenddate()) .ofNullable(input.getEmbargoenddate())
@ -181,6 +178,10 @@ public class Mapper implements Serializable {
KeyValue KeyValue
.newInstance("subject", s.getQualifier().getClassid() + ":" + s.getValue())))); .newInstance("subject", s.getQualifier().getClassid() + ":" + s.getValue()))));
Optional
.ofNullable(input.getResourcetype())
.ifPresent(value -> externals.add(KeyValue.newInstance("resource type", value.getClassname())));
cfSet.forEach(cf -> externals.add(KeyValue.newInstance("collected from", cf))); cfSet.forEach(cf -> externals.add(KeyValue.newInstance("collected from", cf)));
hbSet.forEach(hb -> externals.add(KeyValue.newInstance("hosted by", hb))); hbSet.forEach(hb -> externals.add(KeyValue.newInstance("hosted by", hb)));
urlSet.forEach(url -> externals.add(KeyValue.newInstance("url", url))); urlSet.forEach(url -> externals.add(KeyValue.newInstance("url", url)));
@ -200,5 +201,4 @@ public class Mapper implements Serializable {
it.forEachRemaining(v -> externals.add(KeyValue.newInstance("description", v.getValue()))); it.forEachRemaining(v -> externals.add(KeyValue.newInstance("description", v.getValue())));
} }
} }

4
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/dump/gcat/software.json
View File

File diff suppressed because one or more lines are too long