MaDgIK
/
openaire-search-api-v2
13
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

[Mapping] alignment to the new dump-schema and the new version of the json payload #5

Merged
aisimeon349e6 merged 1 commits from jsonPayload into master 2024-09-25 10:30:29 +02:00
Conversation 0 Commits 1 Files Changed 11 +178 -62
11 changed files with 178 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 2e895e0144 - Show all commits

4
pom.xml
View File

@ -167,14 +167,14 @@
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-dump-schema</artifactId>
<version>[8.0.0-SNAPSHOT]</version>
<version>[8.0.0]</version>
</dependency>
<!-- Add graph schema dependency -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
<version>[6.1.1]</version>
<version>[6.1.3]</version>
</dependency>
<dependency>
<groupId>junit</groupId>

8
src/main/java/eu/openaire/api/mappers/response/entities/Constants.java
View File

@ -12,6 +12,14 @@ public class Constants {
public static final String CF1CF = "c_f1cf";
public static final String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
public static final String USAGE_COUNT_DOWNLOADS = "downloads";
public static final String USAGE_COUNT_VIEWS = "views";
public static final String BIP_POPULARITY = "popularity";
public static final String BIP_POPULARITY_ALT = "popularity_alt";
public static final String BIP_INFLUENCE = "influence";
public static final String BIP_INFLUENCE_ALT = "influence_alt";
public static final String BIP_IMPULSE = "impulse";
public static final Map<String, String> ACCESS_RIGHTS_COAR_MAP = Maps.newHashMap();
static {

22
src/main/java/eu/openaire/api/mappers/response/entities/DatasourceMapper.java
View File

@ -1,6 +1,7 @@
package eu.openaire.api.mappers.response.entities;
import eu.dnetlib.dhp.oa.model.Container;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.oa.model.graph.Datasource;
import eu.dnetlib.dhp.oa.model.graph.DatasourcePid;
import eu.dnetlib.dhp.oa.model.graph.DatasourceSchemeValue;
@ -15,19 +16,18 @@ import java.util.stream.Collectors;
@Mapper(componentModel = "spring")
public interface DatasourceMapper {
@Mapping(target = "journal", expression = "java( mapJournal(payload.getDatasource().getJournal()) )")
@Mapping(target = "policies", expression = "java( mapPolicies(payload.getDatasource().getPolicies()) )")
@Mapping(target = "subjects", expression = "java( mapSubjects(payload.getDatasource().getSubjects()) )")
@Mapping(target = "openaireCompatibility", source = "payload.datasource.openairecompatibility.label")
@Mapping(target = "originalId", source = "payload.header.originalId")
@Mapping(target = "pid", expression = "java( mapPids(payload.getPid()) )")
@Mapping(target = "datasourceType", expression = "java( mapDatasourceType(payload.getDatasource().getDatasourcetype()))")
@Mapping(target = "id", source = "payload.header.id")
@Mapping(target = "originalId", source = "payload.header.originalId")
@Mapping(target = "type", expression = "java( mapDatasourceType(payload.getDatasource().getDatasourcetype()))")
@Mapping(target = "openaireCompatibility", source = "payload.datasource.openairecompatibility.label")
@Mapping(target = "officialName", source = "payload.datasource.officialname")
@Mapping(target = "englishName", source = "payload.datasource.englishname")
@Mapping(target = "websiteUrl", source = "payload.datasource.websiteurl")
@Mapping(target = "logoUrl", source = "payload.datasource.logourl")
@Mapping(target = "dateOfValidation", source = "payload.datasource.dateofvalidation")
@Mapping(target = "description", source = "payload.datasource.description")
@Mapping(target = "subjects", expression = "java( mapSubjects(payload.getDatasource().getSubjects()) )")
@Mapping(target = "languages", source = "payload.datasource.odlanguages")
@Mapping(target = "contentTypes", source = "payload.datasource.odcontenttypes")
@Mapping(target = "releaseStartDate", source = "payload.datasource.releasestartdate")
@ -41,7 +41,8 @@ public interface DatasourceMapper {
@Mapping(target = "citationGuidelineUrl", source = "payload.datasource.citationguidelineurl")
@Mapping(target = "pidSystems", source = "payload.datasource.pidsystems")
@Mapping(target = "certificates", source = "payload.datasource.certificates")
@Mapping(target = "id", source = "payload.header.id")
@Mapping(target = "policies", expression = "java( mapPolicies(payload.getDatasource().getPolicies()) )")
@Mapping(target = "journal", expression = "java( mapJournal(payload.getDatasource().getJournal()) )")
Datasource toGraphDatasource(SolrRecord payload);
@Named("mapDatasourceType")
@ -49,7 +50,7 @@ public interface DatasourceMapper {
@Mapping(target = "value", source = "label")
DatasourceSchemeValue mapDatasourceType(CodeLabel codeLabel) ;
@Mapping(target = "scheme", source = "type")
@Mapping(target = "scheme", source = "typeCode")
DatasourcePid mapPid(Pid pid);
@Named("mapPids")
@ -80,5 +81,10 @@ public interface DatasourceMapper {
Container mapJournal(Journal journal );
@Named("mapIndicators")
default Indicator mapIndicators(List<Measure> measureList){
return Utils.mapIndicators(measureList);
}
}

8
src/main/java/eu/openaire/api/mappers/response/entities/OrganizationMapper.java
View File

@ -1,9 +1,11 @@
package eu.openaire.api.mappers.response.entities;
import eu.dnetlib.dhp.oa.model.Country;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.oa.model.graph.Organization;
import eu.dnetlib.dhp.oa.model.graph.OrganizationPid;
import eu.dnetlib.dhp.schema.solr.CodeLabel;
import eu.dnetlib.dhp.schema.solr.Measure;
import eu.dnetlib.dhp.schema.solr.Pid;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
import org.mapstruct.Mapper;
@ -29,7 +31,7 @@ public interface OrganizationMapper {
@Named("mapCountry")
Country mapCountry(CodeLabel codeLabel) ;
@Mapping(target = "scheme", source = "type")
@Mapping(target = "scheme", source = "typeCode")
OrganizationPid mapPid(Pid pid);
@Named("mapPids")
@ -39,5 +41,9 @@ public interface OrganizationMapper {
return pids.stream().map(this::mapPid).collect(Collectors.toList());
}
@Named("mapIndicators")
default Indicator mapIndicators(List<Measure> measureList){
return Utils.mapIndicators(measureList);
}
}

12
src/main/java/eu/openaire/api/mappers/response/entities/ProjectMapper.java
View File

@ -1,13 +1,11 @@
package eu.openaire.api.mappers.response.entities;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.oa.model.graph.Funder;
import eu.dnetlib.dhp.oa.model.graph.Fundings;
import eu.dnetlib.dhp.oa.model.graph.Granted;
import eu.dnetlib.dhp.oa.model.graph.Project;
import eu.dnetlib.dhp.schema.solr.Funding;
import eu.dnetlib.dhp.schema.solr.FundingLevel;
import eu.dnetlib.dhp.schema.solr.SolrRecord;
import eu.dnetlib.dhp.schema.solr.Subject;
import eu.dnetlib.dhp.schema.solr.*;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.Named;
@ -18,7 +16,7 @@ import java.util.stream.Collectors;
@Mapper(componentModel = "spring")
public interface ProjectMapper {
//no indicator for project are present in the dump. Consider to include them
@Mapping(target = "id", source = "payload.header.id")
@Mapping(target = "websiteUrl", source = "payload.project.websiteurl")
@Mapping(target = "acronym", source = "payload.project.acronym")
@ -92,5 +90,9 @@ public interface ProjectMapper {
return fundings;
}
@Named("mapIndicators")
default Indicator mapIndicators(List<Measure> measureList){
return Utils.mapIndicators(measureList);
}
}

82
src/main/java/eu/openaire/api/mappers/response/entities/ResearchProductMapper.java
View File

@ -11,6 +11,7 @@ import eu.dnetlib.dhp.schema.solr.BestAccessRight;
import eu.dnetlib.dhp.schema.solr.Country;
import eu.dnetlib.dhp.schema.solr.Instance;
import eu.dnetlib.dhp.schema.solr.Language;
import eu.dnetlib.dhp.schema.solr.Measure;
import eu.dnetlib.dhp.schema.solr.OpenAccessColor;
import eu.dnetlib.dhp.schema.solr.Subject;
import eu.dnetlib.dhp.schema.solr.*;
@ -24,45 +25,52 @@ import java.util.stream.Collectors;
@Mapper(componentModel = "spring")
public interface ResearchProductMapper {
//@Mapping(target = "indicators", expression = "java (mapAuthor(payload.getResult().getIndicators()))") missing in the payload
@Mapping(target = "bestAccessRight", expression = "java(mapBestAccessRight(payload.getResult().getBestaccessright()))")
@Mapping(target = "instance", expression = "java(mapInstances(payload.getResult().getInstance()))")
@Mapping(target = "author", expression = "java(mapAuthors(payload.getResult().getAuthor()))")
@Mapping(target = "openAccessColor", expression = "java(mapOpenAccessColor(payload.getResult().getOpenAccessColor()))")
@Mapping(target = "green", source = "payload.result.green")
@Mapping(target = "inDiamondJournal", source = "payload.result.inDiamondJournal")
@Mapping(target = "publiclyFunded", source = "payload.result.publiclyFunded")
@Mapping(target = "type", source = "payload.result.resulttype")
@Mapping(target = "country", expression = "java(mapCountries(payload.getResult().getCountry()))")
@Mapping(target = "id", source = "payload.header.id")
//From the model of the dump we miss the fulltext and the eoscIFGuidelines, otherTitles, transformativeAgreement. We need to consider if we want to include them in the dump
@Mapping(target = "pid", expression = "java(mapPids(payload.getPid()) )")
@Mapping(target = "indicators", expression = "java(mapIndicators(payload.getMeasures()))")
@Mapping(target = "id", source = "payload.header.id")
@Mapping(target = "originalId", source = "payload.header.originalId")
@Mapping(target = "version", source = "payload.result.version")
@Mapping(target = "size", source = "payload.result.size")
@Mapping(target = "tool", source = "payload.result.tool")
@Mapping(target = "contactGroup", source = "payload.result.contactgroup")
@Mapping(target = "contactPerson", source = "payload.result.contactperson")
@Mapping(target = "programmingLanguage", source = "payload.result.programmingLanguage")
@Mapping(target = "codeRepositoryUrl", source = "payload.result.codeRepositoryUrl")
@Mapping(target = "documentationUrl", source = "payload.result.documentationUrl")
@Mapping(target = "container", expression = "java(mapJournal(payload.getResult().getJournal()) )")
@Mapping(target = "coverage", source = "payload.result.coverage")
@Mapping(target = "contributor", source = "payload.result.contributor")
@Mapping(target = "format", source = "payload.result.format")
@Mapping(target = "source", source = "payload.result.source")
@Mapping(target = "embargoEndDate", source = "payload.result.embargoenddate")
@Mapping(target = "publisher", source = "payload.result.publisher")
@Mapping(target = "publicationDate", source = "payload.result.publicationdate")
@Mapping(target = "description", source = "payload.result.description")
@Mapping(target = "mainTitle", source = "payload.result.maintitle")
@Mapping(target = "type", source = "payload.result.resulttype")
@Mapping(target = "author", expression = "java(mapAuthors(payload.getResult().getAuthor()))")
@Mapping(target = "subjects", expression = "java(mapSubjects(payload.getResult().getSubject()) )")
@Mapping(target = "language", expression = "java(mapLanguage(payload.getResult().getLanguage()))")
@Mapping(target = "country", expression = "java(mapCountries(payload.getResult().getCountry()))")
@Mapping(target = "mainTitle", source = "payload.result.maintitle")
@Mapping(target = "description", source = "payload.result.description")
@Mapping(target = "publicationDate", source = "payload.result.publicationdate")
@Mapping(target = "publisher", source = "payload.result.publisher")
@Mapping(target = "embargoEndDate", source = "payload.result.embargoenddate")
@Mapping(target = "source", source = "payload.result.source")
@Mapping(target = "format", source = "payload.result.format")
@Mapping(target = "contributor", source = "payload.result.contributor")
@Mapping(target = "coverage", source = "payload.result.coverage")
@Mapping(target = "bestAccessRight", expression = "java(mapBestAccessRight(payload.getResult().getBestaccessright()))")
@Mapping(target = "container", expression = "java(mapJournal(payload.getResult().getJournal()) )")
@Mapping(target = "documentationUrl", source = "payload.result.documentationUrl")
@Mapping(target = "codeRepositoryUrl", source = "payload.result.codeRepositoryUrl")
@Mapping(target = "programmingLanguage", source = "payload.result.programmingLanguage")
@Mapping(target = "contactPerson", source = "payload.result.contactperson")
@Mapping(target = "contactGroup", source = "payload.result.contactgroup")
@Mapping(target = "tool", source = "payload.result.tool")
@Mapping(target = "size", source = "payload.result.size")
@Mapping(target = "version", source = "payload.result.version")
@Mapping(target = "isGreen", source = "payload.result.isGreen")
@Mapping(target = "openAccessColor", expression = "java(mapOpenAccessColor(payload.getResult().getOpenAccessColor()))")
@Mapping(target = "isInDiamondJournal", source = "payload.result.isInDiamondJournal")
@Mapping(target = "publiclyFunded", source = "payload.result.publiclyFunded")
@Mapping(target = "instance", expression = "java(mapInstances(payload.getResult().getInstance()))")
GraphResult toGraphResult(SolrRecord payload);
@Mapping(target = "scheme", source = "type")
@Mapping(target = "scheme", source = "typeCode")
ResultPid mapPid(Pid pid);
@Named("mapIndicators")
default Indicator mapIndicators(List<Measure> measureList){
return Utils.mapIndicators(measureList);
}
@Named("mapPids")
default List<ResultPid> mapPids(List<Pid> pids ){
if(pids == null)
@ -77,7 +85,7 @@ public interface ResearchProductMapper {
return altIdentifiers.stream().map(this::mapAltId).collect(Collectors.toList());
}
@Mapping(target = "scheme", source = "type")
@Mapping(target = "scheme", source = "typeCode")
AlternateIdentifier mapAltId(Pid altId);
@Named("mapJournal")
@ -96,7 +104,7 @@ public interface ResearchProductMapper {
eu.dnetlib.dhp.oa.model.Subject mapSubject(Subject subject);
@Named("mapSubjectSchemeValue")
@Mapping(target = "scheme", source = "type")
@Mapping(target = "scheme", source = "typeCode")
SubjectSchemeValue mapSubjectSchemeValue(Subject subject);
@Named("mapCountries")
@ -133,7 +141,7 @@ public interface ResearchProductMapper {
.stream()
.filter(
ap -> ap
.getType()
.getTypeCode()
.equals(ModelConstants.ORCID))
.collect(Collectors.toList());
if (orcid.size() == 1) {
@ -145,7 +153,7 @@ public interface ResearchProductMapper {
.stream()
.filter(
ap -> ap
.getType()
.getTypeCode()
.equals(ModelConstants.ORCID_PENDING))
.collect(Collectors.toList());
if (orcid.size() == 1) {
@ -159,7 +167,7 @@ public interface ResearchProductMapper {
AuthorPid getAuthorPid(Pid pid);
@Named("mapAuthorPidSchemeValue")
@Mapping(target = "scheme", source = "pid.type")
@Mapping(target = "scheme", source = "pid.typeCode")
AuthorPidSchemeValue mapAuthorPidSchemeValue (Pid pid);
@Named("mapBestAccessRight")
@ -199,8 +207,8 @@ public interface ResearchProductMapper {
if(accessright==null)
return null;
AccessRight ar = new AccessRight();
if (Constants.ACCESS_RIGHTS_COAR_MAP.containsKey(accessright.getValue())) {
String code = Constants.ACCESS_RIGHTS_COAR_MAP.get(accessright.getValue());
if (Constants.ACCESS_RIGHTS_COAR_MAP.containsKey(accessright.getLabel())) {
String code = Constants.ACCESS_RIGHTS_COAR_MAP.get(accessright.getLabel());
ar.setCode(code);
ar.setLabel(Constants.COAR_CODE_LABEL_MAP.get(code));
ar.setScheme(Constants.COAR_ACCESS_RIGHT_SCHEMA);

88
src/main/java/eu/openaire/api/mappers/response/entities/Utils.java Normal file
View File

@ -0,0 +1,88 @@
package eu.openaire.api.mappers.response.entities;
import eu.dnetlib.dhp.oa.model.AuthorPid;
import eu.dnetlib.dhp.oa.model.BipIndicators;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.oa.model.UsageCounts;
import eu.dnetlib.dhp.oa.model.graph.DatasourcePid;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.solr.Measure;
import eu.dnetlib.dhp.schema.solr.Pid;
import java.util.List;
import java.util.stream.Collectors;
public class Utils {
public static Indicator mapIndicators(List<Measure> measureList){
if(measureList == null)
return null;
Indicator i = new Indicator();
for (Measure m : measureList) {
switch (m.getId()) {
case Constants.USAGE_COUNT_DOWNLOADS:
getUsageCounts(i).setDownloads(Integer.parseInt(m.getUnit().get(0).getLabel()));
break;
case Constants.USAGE_COUNT_VIEWS:
getUsageCounts(i).setViews(Integer.parseInt(m.getUnit().get(0).getLabel()));
break;
case Constants.BIP_INFLUENCE:
m.getUnit().forEach(u -> {
if (u.getCode().equals("class"))
getImpactMeasure(i).setInfluenceClass(u.getLabel());
if (u.getCode().equals("score"))
getImpactMeasure(i).setInfluence(Double.parseDouble(u.getLabel()));
});
break;
case Constants.BIP_POPULARITY:
m.getUnit().forEach(u -> {
if (u.getCode().equals("class"))
getImpactMeasure(i).setPopularityClass(u.getLabel());
if (u.getCode().equals("score"))
getImpactMeasure(i).setPopularity(Double.parseDouble(u.getLabel()));
});
break;
case Constants.BIP_INFLUENCE_ALT:
m.getUnit().forEach(u -> {
if (u.getCode().equals("class"))
getImpactMeasure(i).setCitationClass(u.getLabel());
if (u.getCode().equals("score"))
getImpactMeasure(i).setCitationCount(Double.parseDouble(u.getLabel()));
});
break;
case Constants.BIP_POPULARITY_ALT:
break;
case Constants.BIP_IMPULSE:
m.getUnit().forEach(u -> {
if (u.getCode().equals("class"))
getImpactMeasure(i).setImpulseClass(u.getLabel());
if (u.getCode().equals("score"))
getImpactMeasure(i).setImpulse(Double.parseDouble(u.getLabel()));
});
break;
default:
throw new RuntimeException("No mapping found for indicator " + m.getId());
}
}
return i;
}
private static UsageCounts getUsageCounts(Indicator i) {
if (i.getUsageCounts() == null) {
i.setUsageCounts(new UsageCounts());
}
return i.getUsageCounts();
}
private static BipIndicators getImpactMeasure(Indicator i) {
if (i.getCitationImpact() == null) {
i.setCitationImpact(new BipIndicators());
}
return i.getCitationImpact();
}
}

4
src/test/java/eu/openaire/api/mappers/response/DatasourceMapperTest.java
View File

@ -36,8 +36,8 @@ public class DatasourceMapperTest {
Assertions.assertEquals("doajarticles::614fdb5f82725ed3f8834ae90b9a0212", graphDatasource.getId());
Assertions.assertEquals(2, graphDatasource.getOriginalId().size());
Assertions.assertTrue(graphDatasource.getOriginalId().containsAll(Arrays.asList("doajarticles::2196-8403", "issn___print::2196-8403")));
Assertions.assertEquals("Journal", graphDatasource.getDatasourceType().getValue());
Assertions.assertEquals("pubsrepository::journal", graphDatasource.getDatasourceType().getScheme());
Assertions.assertEquals("Journal", graphDatasource.getType().getValue());
Assertions.assertEquals("pubsrepository::journal", graphDatasource.getType().getScheme());
Assertions.assertEquals("collected from a compatible aggregator", graphDatasource.getOpenaireCompatibility());
Assertions.assertEquals("Convivium", graphDatasource.getOfficialName());
Assertions.assertEquals("Convivium", graphDatasource.getEnglishName());

4
src/test/java/eu/openaire/api/mappers/response/ResultProductMapperTest.java
View File

@ -75,8 +75,8 @@ public class ResultProductMapperTest {
Assertions.assertEquals(Constants.COAR_ACCESS_RIGHT_SCHEMA, graphResult.getBestAccessRight().getScheme());
Assertions.assertFalse(graphResult.getPubliclyFunded());
Assertions.assertFalse(graphResult.getGreen());
Assertions.assertFalse(graphResult.getInDiamondJournal());
Assertions.assertFalse(graphResult.getIsGreen());
Assertions.assertFalse(graphResult.getIsInDiamondJournal());
Assertions.assertEquals(1, graphResult.getInstance().size());
Instance instance = graphResult.getInstance().get(0);

2
src/test/resources/eu/openaire/api/mappers/response/SolrDatasource.json
View File

@ -55,7 +55,7 @@
"subjects": [
{
"value": "Language and Literature: German literature | Language and Literature: Philology. Linguistics",
"type": "keyword"
"typeCode": "keyword"
}
],
"odnumberofitems": "0.0",

6
src/test/resources/eu/openaire/api/mappers/response/SolrResult.json
View File

@ -80,7 +80,7 @@
"rank": 11,
"pid": [
{
"type": "orcid_pending",
"typeCode": "orcid_pending",
"value": "0000-0002-0425-5364"
}
]
@ -95,7 +95,7 @@
"publiclyFunded": false,
"instance": [
{
"accessright": {"value": "CLOSED"},
"accessright": {"label": "CLOSED"},
"instancetype": "0013",
"hostedby": {
"dsId": "10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747",
@ -110,8 +110,6 @@
"refereed": "0002"
}
],
"green": false,
"inDiamondJournal": false,
"isGreen": false,
"isInDiamondJournal": false
}