sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[graph provision] upgrade to dhp-schema:6.1.2, included project.oamandatepublications in the JSON payload mapping, fixed serialisation of the usageCounts measures

This commit is contained in:
Claudio Atzori 2024-05-14 12:39:31 +02:00
parent 53e7bb4336
commit 1efe7f7e39
5 changed files with 49 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java
View File

@ -65,9 +65,9 @@ public class PayloadConverterJob {
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
final Boolean validateXML = Optional final Boolean validateXML = Optional
.ofNullable(parser.get("validateXML")) .ofNullable(parser.get("validateXML"))
.map(Boolean::valueOf) .map(Boolean::valueOf)
.orElse(Boolean.FALSE); .orElse(Boolean.FALSE);
log.info("validateXML: {}", validateXML); log.info("validateXML: {}", validateXML);
final String contextApiBaseUrl = parser.get("contextApiBaseUrl"); final String contextApiBaseUrl = parser.get("contextApiBaseUrl");

1
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java
View File

@ -146,6 +146,7 @@ public class ProvisionModelSupport {
ps.setContracttype(mapCodeLabel(p.getContracttype())); ps.setContracttype(mapCodeLabel(p.getContracttype()));
ps.setCurrency(mapField(p.getCurrency())); ps.setCurrency(mapField(p.getCurrency()));
ps.setDuration(mapField(p.getDuration())); ps.setDuration(mapField(p.getDuration()));
ps.setOamandatepublications(mapField(p.getOamandatepublications()));
ps.setCallidentifier(mapField(p.getCallidentifier())); ps.setCallidentifier(mapField(p.getCallidentifier()));
ps.setEcarticle29_3(mapField(p.getEcarticle29_3())); ps.setEcarticle29_3(mapField(p.getEcarticle29_3()));
ps.setEnddate(mapField(p.getEnddate())); ps.setEnddate(mapField(p.getEnddate()));

79
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java
View File

@ -1,23 +1,25 @@
package eu.dnetlib.dhp.oa.provision.utils; package eu.dnetlib.dhp.oa.provision.utils;
import com.fasterxml.jackson.databind.ObjectMapper; import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes;
import com.google.common.base.Joiner; import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor;
import com.google.common.base.Splitter; import static org.apache.commons.lang3.StringUtils.isNotBlank;
import com.google.common.collect.Lists; import static org.apache.commons.lang3.StringUtils.substringBefore;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets; import java.io.IOException;
import com.mycila.xmltool.XMLDoc; import java.io.Serializable;
import com.mycila.xmltool.XMLTag; import java.io.StringReader;
import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; import java.io.StringWriter;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; import java.net.MalformedURLException;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; import java.net.URL;
import eu.dnetlib.dhp.oa.provision.model.XmlInstance; import java.util.*;
import eu.dnetlib.dhp.schema.common.*; import java.util.stream.Collectors;
import eu.dnetlib.dhp.schema.oaf.Result; import java.util.stream.Stream;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import javax.xml.transform.*;
import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
@ -29,26 +31,27 @@ import org.dom4j.Node;
import org.dom4j.io.OutputFormat; import org.dom4j.io.OutputFormat;
import org.dom4j.io.SAXReader; import org.dom4j.io.SAXReader;
import org.dom4j.io.XMLWriter; import org.dom4j.io.XMLWriter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.mycila.xmltool.XMLDoc;
import com.mycila.xmltool.XMLTag;
import eu.dnetlib.dhp.oa.provision.model.JoinedEntity;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntity;
import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper;
import eu.dnetlib.dhp.oa.provision.model.XmlInstance;
import eu.dnetlib.dhp.schema.common.*;
import eu.dnetlib.dhp.schema.oaf.*;
import eu.dnetlib.dhp.schema.oaf.Result;
import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory;
import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits;
import scala.Tuple2; import scala.Tuple2;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.IOException;
import java.io.Serializable;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes;
import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.substringBefore;
public class XmlRecordFactory implements Serializable { public class XmlRecordFactory implements Serializable {
/** /**
@ -127,9 +130,9 @@ public class XmlRecordFactory implements Serializable {
if (Boolean.TRUE.equals(validate)) { if (Boolean.TRUE.equals(validate)) {
// rise an exception when an invalid record was built // rise an exception when an invalid record was built
new SAXReader().read(new StringReader(xmlRecord)); new SAXReader().read(new StringReader(xmlRecord));
} }
return xmlRecord; return xmlRecord;
// return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent); // return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent);
} catch (final Throwable e) { } catch (final Throwable e) {
throw new RuntimeException(String.format("error building record '%s'", entity.getId()), e); throw new RuntimeException(String.format("error building record '%s'", entity.getId()), e);
} }

8
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java
View File

@ -190,11 +190,9 @@ public class XmlSerializationUtils {
sb.append(" ").append(attr(measure.getId(), kv.getValue())); sb.append(" ").append(attr(measure.getId(), kv.getValue()));
} }
sb sb
.append(">") .append(" ")
.append(dsId) .append(attr("datasource", dsId))
.append("</") .append("/>");
.append(name)
.append(">");
}); });
return sb.toString(); return sb.toString();
} }

2
pom.xml
View File

@ -888,7 +888,7 @@
<mockito-core.version>3.3.3</mockito-core.version> <mockito-core.version>3.3.3</mockito-core.version>
<mongodb.driver.version>3.4.2</mongodb.driver.version> <mongodb.driver.version>3.4.2</mongodb.driver.version>
<vtd.version>[2.12,3.0)</vtd.version> <vtd.version>[2.12,3.0)</vtd.version>
<dhp-schemas.version>[6.1.1]</dhp-schemas.version> <dhp-schemas.version>[6.1.2]</dhp-schemas.version>
<dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version> <dnet-actionmanager-api.version>[4.0.3]</dnet-actionmanager-api.version>
<dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version> <dnet-actionmanager-common.version>[6.0.5]</dnet-actionmanager-common.version>
<dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version> <dnet-openaire-broker-common.version>[3.1.6]</dnet-openaire-broker-common.version>