D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 11 Projects 2 Releases 2 Wiki Activity

Align Solr JSON records to the explore portal requirements #448

Merged
claudio.atzori merged 4 commits from json_payload into beta_to_master_may2024 2024-06-25 09:57:40 +02:00
Conversation 0 Commits 4 Files Changed 6 +60 -15
6 changed files with 60 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java
View File

@ -5,6 +5,7 @@ import java.io.StringReader;
import java.util.*;
import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.solr.Measure;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.Document;
import org.dom4j.DocumentException;
@ -76,6 +77,7 @@ public class ProvisionModelSupport {
r.setCollectedfrom(asProvenance(e.getCollectedfrom()));
r.setContext(asContext(e.getContext(), contextMapper));
r.setPid(asPid(e.getPid()));
r.setMeasures(mapMeasures(e.getMeasures()));
if (e instanceof eu.dnetlib.dhp.schema.oaf.Result) {
r.setResult(mapResult((eu.dnetlib.dhp.schema.oaf.Result) e));
@ -106,6 +108,13 @@ public class ProvisionModelSupport {
final RelatedEntity re = rew.getTarget();
final RecordType relatedRecordType = RecordType.valueOf(re.getType());
final Relation relation = rew.getRelation();
final String relationProvenance = Optional
.ofNullable(relation.getDataInfo())
.map(d -> Optional
.ofNullable(d.getProvenanceaction())
.map(Qualifier::getClassid)
.orElse(null))
.orElse(null);
rr
.setHeader(
RelatedRecordHeader
@ -113,7 +122,9 @@ public class ProvisionModelSupport {
relation.getRelType(),
relation.getRelClass(),
StringUtils.substringAfter(relation.getTarget(), IdentifierFactory.ID_PREFIX_SEPARATOR),
relatedRecordType));
relatedRecordType,
relationProvenance,
Optional.ofNullable(relation.getDataInfo()).map(DataInfo::getTrust).orElse(null)));
rr.setAcronym(re.getAcronym());
rr.setCode(re.getCode());
@ -131,11 +142,19 @@ public class ProvisionModelSupport {
rr.setOfficialname(re.getOfficialname());
rr.setOpenairecompatibility(mapCodeLabel(re.getOpenairecompatibility()));
rr.setPid(asPid(re.getPid()));
rr.setProjectTitle(rr.getProjectTitle());
rr.setWebsiteurl(re.getWebsiteurl());
rr.setProjectTitle(re.getProjectTitle());
rr.setPublisher(re.getPublisher());
rr.setResulttype(mapQualifier(re.getResulttype()));
rr.setTitle(Optional.ofNullable(re.getTitle()).map(StructuredProperty::getValue).orElse(null));
if (relation.getValidated() == null) {
relation.setValidated(false);
}
if (ModelConstants.OUTCOME.equals(relation.getSubRelType()) && StringUtils.isNotBlank(relation.getValidationDate())) {
rr.setValidationDate(relation.getValidationDate());
}
return rr;
}
@ -266,6 +285,7 @@ public class ProvisionModelSupport {
ds.setOfficialname(mapField(d.getOfficialname()));
ds.setDescription(mapField(d.getDescription()));
ds.setJournal(mapJournal(d.getJournal()));
ds.setWebsiteurl(mapField(d.getWebsiteurl()));
ds.setLogourl(mapField(d.getLogourl()));
ds.setAccessinfopackage(mapFieldList(d.getAccessinfopackage()));
ds.setCertificates(mapField(d.getCertificates()));
@ -311,6 +331,7 @@ public class ProvisionModelSupport {
ds.setSubjects(asSubjectSP(d.getSubjects()));
ds.setSubmissionpolicyurl(d.getSubmissionpolicyurl());
ds.setThematic(d.getThematic());
ds.setContentpolicies(mapCodeLabel(d.getContentpolicies()));
ds.setVersioncontrol(d.getVersioncontrol());
ds.setVersioning(mapField(d.getVersioning()));
@ -326,6 +347,7 @@ public class ProvisionModelSupport {
rs.setOtherTitles(getOtherTitles(r.getTitle()));
rs.setDescription(mapFieldList(r.getDescription()));
rs.setSubject(asSubject(r.getSubject()));
rs.setLanguage(asLanguage(r.getLanguage()));
rs.setPublicationdate(mapField(r.getDateofacceptance()));
rs.setPublisher(mapField(r.getPublisher()));
rs.setEmbargoenddate(mapField(r.getEmbargoenddate()));
@ -375,6 +397,12 @@ public class ProvisionModelSupport {
return rs;
}
private static Language asLanguage(Qualifier lang) {
return Optional.ofNullable(lang)
.map(q -> Language.newInstance(q.getClassid(), q.getClassname()))
.orElse(null);
}
@Nullable
private static List<String> getOtherTitles(List<StructuredProperty> titleList) {
return Optional
@ -422,7 +450,7 @@ public class ProvisionModelSupport {
Instance i = new Instance();
i.setCollectedfrom(asProvenance(instance.getCollectedfrom()));
i.setHostedby(asProvenance(instance.getHostedby()));
i.setFulltext(i.getFulltext());
i.setFulltext(instance.getFulltext());
i.setPid(asPid(instance.getPid()));
i.setAlternateIdentifier(asPid(instance.getAlternateIdentifier()));
i.setAccessright(mapAccessRight(instance.getAccessright()));
@ -453,7 +481,8 @@ public class ProvisionModelSupport {
private static AccessRight mapAccessRight(eu.dnetlib.dhp.schema.oaf.AccessRight accessright) {
return AccessRight
.newInstance(
mapQualifier(accessright),
accessright.getClassid(),
accessright.getClassname(),
Optional
.ofNullable(accessright.getOpenAccessRoute())
.map(route -> OpenAccessRoute.valueOf(route.toString()))
@ -508,7 +537,18 @@ public class ProvisionModelSupport {
}
private static Provenance asProvenance(KeyValue keyValue) {
return Optional.ofNullable(keyValue).map(cf -> Provenance.newInstance(cf.getKey(), cf.getValue())).orElse(null);
return Optional.ofNullable(keyValue).map(kv ->
Provenance.newInstance(
StringUtils.substringAfter(kv.getKey(), IdentifierFactory.ID_PREFIX_SEPARATOR),
kv.getValue())).orElse(null);
}
private static List<Measure> mapMeasures(List<eu.dnetlib.dhp.schema.oaf.Measure> measures) {
return Optional.ofNullable(measures)
.map(ml -> ml.stream()
.map(m -> Measure.newInstance(m.getId(), mapCodeLabelKV(m.getUnit())))
.collect(Collectors.toList()))
.orElse(null);
}
private static List<Context> asContext(List<eu.dnetlib.dhp.schema.oaf.Context> ctxList,
@ -581,7 +621,12 @@ public class ProvisionModelSupport {
.map(
pids -> pids
.stream()
.map(p -> Pid.newInstance(p.getQualifier().getClassname(), p.getValue()))
.filter(p -> Objects.nonNull(p.getQualifier()))
.filter(p -> Objects.nonNull(p.getQualifier().getClassid()))
.map(p -> Pid.newInstance(
p.getValue(),
p.getQualifier().getClassid(),
p.getQualifier().getClassname()))
.collect(Collectors.toList()))
.orElse(null);
}
@ -607,7 +652,7 @@ public class ProvisionModelSupport {
.stream()
.filter(s -> Objects.nonNull(s.getQualifier()))
.filter(s -> Objects.nonNull(s.getQualifier().getClassname()))
.map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassname()))
.map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassid(), s.getQualifier().getClassname()))
.collect(Collectors.toList()))
.orElse(null);
}
@ -620,7 +665,7 @@ public class ProvisionModelSupport {
.stream()
.filter(s -> Objects.nonNull(s.getQualifier()))
.filter(s -> Objects.nonNull(s.getQualifier().getClassname()))
.map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassname()))
.map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassid(), s.getQualifier().getClassname()))
.collect(Collectors.toList()))
.orElse(null);
}
@ -689,7 +734,7 @@ public class ProvisionModelSupport {
private static CodeLabel mapCodeLabel(KeyValue kv) {
return Optional
.ofNullable(kv)
.map(q -> CodeLabel.newInstance(kv.getKey(), kv.getValue()))
.map(k -> CodeLabel.newInstance(k.getKey(), k.getValue()))
.orElse(null);
}

2
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/StreamingInputDocumentFactory.java
View File

@ -213,7 +213,7 @@ public class StreamingInputDocumentFactory implements Serializable {
}
writer.close();
indexDocument.addField(INDEX_RESULT, results.toString());
// indexDocument.addField(INDEX_JSON_RESULT, json);
indexDocument.addField(INDEX_JSON_RESULT, json);
} finally {
outputFactory.remove();
eventFactory.remove();

8
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java
View File

@ -115,12 +115,12 @@ public class XmlIndexingJobTest extends SolrTest {
rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isgreen:true"));
assertEquals(
0, rsp.getResults().getNumFound(),
4, rsp.getResults().getNumFound(),
"the number of indexed records having isgreen = true");
rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "openaccesscolor:bronze"));
assertEquals(
0, rsp.getResults().getNumFound(),
2, rsp.getResults().getNumFound(),
"the number of indexed records having openaccesscolor = bronze");
rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isindiamondjournal:true"));
@ -135,14 +135,14 @@ public class XmlIndexingJobTest extends SolrTest {
rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "peerreviewed:true"));
assertEquals(
0, rsp.getResults().getNumFound(),
35, rsp.getResults().getNumFound(),
"the number of indexed records having peerreviewed = true");
rsp = miniCluster
.getSolrClient()
.query(
new SolrQuery()
.add(CommonParams.Q, "objidentifier:\"iddesignpres::ae77e56e84ad058d9e7f19fa2f7325db\"")
.add(CommonParams.Q, "objidentifier:\"57a035e5b1ae::236d6d8c1e03368b5ae72acfeeb11bbc\"")
.add(CommonParams.FL, "__json"));
assertEquals(
1, rsp.getResults().getNumFound(),

BIN
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/xml/part-00000.json.gz
View File

Binary file not shown.

BIN
dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/xml/xml_json_sample.json.gz Normal file
View File

Binary file not shown.

2
pom.xml
View File

@ -888,7 +888,7 @@
<mockito-core.version>3.3.3</mockito-core.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version>
<vtd.version>[2.12,3.0)</vtd.version>
<dhp-schemas.version>[6.1.2]</dhp-schemas.version>
<dhp-schemas.version>[6.1.3]</dhp-schemas.version>
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>