przemyslaw.jacewicz
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added passing of the following parameters: 

-varDataSourceId
-varOfficialName

in Each transformation Rule
This commit is contained in:
Sandro La Bruzzo 2021-05-11 15:05:58 +02:00
parent d4c3476152
commit 073dcea2aa
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/xslt/XSLTTransformationFunction.java
View File

@ -20,6 +20,10 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
public final static String QNAME_BASE_URI = "http://eu/dnetlib/transform"; public final static String QNAME_BASE_URI = "http://eu/dnetlib/transform";
private final static String DATASOURCE_ID_PARAM="varDataSourceId";
private final static String DATASOURCE_NAME_PARAM="varOfficialName";
private final AggregationCounter aggregationCounter; private final AggregationCounter aggregationCounter;
private final String transformationRule; private final String transformationRule;
@ -48,11 +52,16 @@ public class XSLTTransformationFunction implements MapFunction<MetadataRecord, M
aggregationCounter.getTotalItems().add(1); aggregationCounter.getTotalItems().add(1);
try { try {
Processor processor = new Processor(false); Processor processor = new Processor(false);
processor.registerExtensionFunction(cleanFunction); processor.registerExtensionFunction(cleanFunction);
processor.registerExtensionFunction(new DateCleaner()); processor.registerExtensionFunction(new DateCleaner());
processor.registerExtensionFunction(new PersonCleaner()); processor.registerExtensionFunction(new PersonCleaner());
final XsltCompiler comp = processor.newXsltCompiler(); final XsltCompiler comp = processor.newXsltCompiler();
QName datasourceIDParam = new QName(DATASOURCE_ID_PARAM);
comp.setParameter(datasourceIDParam, new XdmAtomicValue(value.getProvenance().getDatasourceId()));
QName datasourceNameParam = new QName(DATASOURCE_NAME_PARAM);
comp.setParameter(datasourceNameParam, new XdmAtomicValue(value.getProvenance().getDatasourceName()));
XsltExecutable xslt = comp XsltExecutable xslt = comp
.compile(new StreamSource(IOUtils.toInputStream(transformationRule, StandardCharsets.UTF_8))); .compile(new StreamSource(IOUtils.toInputStream(transformationRule, StandardCharsets.UTF_8)));
XdmNode source = processor XdmNode source = processor

6
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/transformation/TransformationJobTest.java
View File

@ -10,6 +10,7 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import eu.dnetlib.dhp.schema.mdstore.Provenance;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
@ -63,6 +64,8 @@ public class TransformationJobTest extends AbstractVocabularyTest {
// We Set the input Record getting the XML from the classpath // We Set the input Record getting the XML from the classpath
final MetadataRecord mr = new MetadataRecord(); final MetadataRecord mr = new MetadataRecord();
mr.setProvenance(new Provenance("DSID", "DSNAME", "PREFIX"));
mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_zenodo.xml"))); mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_zenodo.xml")));
// We Load the XSLT transformation Rule from the classpath // We Load the XSLT transformation Rule from the classpath
XSLTTransformationFunction tr = loadTransformationRule("/eu/dnetlib/dhp/transform/zenodo_tr.xslt"); XSLTTransformationFunction tr = loadTransformationRule("/eu/dnetlib/dhp/transform/zenodo_tr.xslt");
@ -80,6 +83,7 @@ public class TransformationJobTest extends AbstractVocabularyTest {
// We Set the input Record getting the XML from the classpath // We Set the input Record getting the XML from the classpath
final MetadataRecord mr = new MetadataRecord(); final MetadataRecord mr = new MetadataRecord();
mr.setProvenance(new Provenance("DSID", "DSNAME", "PREFIX"));
mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml"))); mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml")));
// We Load the XSLT transformation Rule from the classpath // We Load the XSLT transformation Rule from the classpath
XSLTTransformationFunction tr = loadTransformationRule("/eu/dnetlib/dhp/transform/zenodo_tr.xslt"); XSLTTransformationFunction tr = loadTransformationRule("/eu/dnetlib/dhp/transform/zenodo_tr.xslt");
@ -101,6 +105,7 @@ public class TransformationJobTest extends AbstractVocabularyTest {
// We Set the input Record getting the XML from the classpath // We Set the input Record getting the XML from the classpath
final MetadataRecord mr = new MetadataRecord(); final MetadataRecord mr = new MetadataRecord();
mr.setProvenance(new Provenance("DSID", "DSNAME", "PREFIX"));
mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml"))); mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_itgv4.xml")));
// We Load the XSLT transformation Rule from the classpath // We Load the XSLT transformation Rule from the classpath
XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript); XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript);
@ -121,6 +126,7 @@ public class TransformationJobTest extends AbstractVocabularyTest {
// We Set the input Record getting the XML from the classpath // We Set the input Record getting the XML from the classpath
final MetadataRecord mr = new MetadataRecord(); final MetadataRecord mr = new MetadataRecord();
mr.setProvenance(new Provenance("DSID", "DSNAME", "PREFIX"));
mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_omicsdi.xml"))); mr.setBody(IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input_omicsdi.xml")));
// We Load the XSLT transformation Rule from the classpath // We Load the XSLT transformation Rule from the classpath
XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript); XSLTTransformationFunction tr = loadTransformationRule(xslTransformationScript);