2019-04-11 15:39:29 +02:00
|
|
|
package eu.dnetlib.dhp.transformation;
|
|
|
|
|
|
|
|
import eu.dnetlib.dhp.model.mdstore.MetadataRecord;
|
2019-10-10 11:33:51 +02:00
|
|
|
import eu.dnetlib.dhp.transformation.functions.Cleaner;
|
2019-10-24 11:36:59 +02:00
|
|
|
import eu.dnetlib.dhp.transformation.vocabulary.Vocabulary;
|
|
|
|
import eu.dnetlib.dhp.transformation.vocabulary.VocabularyHelper;
|
2019-04-11 15:39:29 +02:00
|
|
|
import eu.dnetlib.dhp.utils.DHPUtils;
|
2019-10-10 11:33:51 +02:00
|
|
|
import net.sf.saxon.s9api.*;
|
2019-04-11 15:39:29 +02:00
|
|
|
import org.apache.commons.io.IOUtils;
|
|
|
|
import org.apache.spark.util.LongAccumulator;
|
|
|
|
import org.dom4j.Document;
|
|
|
|
import org.dom4j.Node;
|
|
|
|
import org.dom4j.io.SAXReader;
|
2020-03-25 17:59:45 +01:00
|
|
|
import org.junit.jupiter.api.DisplayName;
|
|
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import org.junit.jupiter.api.extension.ExtendWith;
|
|
|
|
import org.junit.jupiter.api.io.TempDir;
|
2019-04-11 15:39:29 +02:00
|
|
|
import org.mockito.Mock;
|
2020-03-25 17:59:45 +01:00
|
|
|
import org.mockito.junit.jupiter.MockitoExtension;
|
2019-04-11 15:39:29 +02:00
|
|
|
|
2019-10-10 11:33:51 +02:00
|
|
|
import javax.xml.transform.stream.StreamSource;
|
|
|
|
import java.io.StringWriter;
|
2019-04-11 15:39:29 +02:00
|
|
|
import java.nio.file.Files;
|
|
|
|
import java.nio.file.Path;
|
2019-10-24 11:36:59 +02:00
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.Map;
|
2019-04-11 15:39:29 +02:00
|
|
|
|
2020-03-25 17:59:45 +01:00
|
|
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
|
|
|
|
|
|
@ExtendWith(MockitoExtension.class)
|
2019-04-11 15:39:29 +02:00
|
|
|
public class TransformationJobTest {
|
2019-11-04 17:41:01 +01:00
|
|
|
|
2019-04-11 15:39:29 +02:00
|
|
|
@Mock
|
2020-03-25 17:59:45 +01:00
|
|
|
private LongAccumulator accumulator;
|
2019-10-24 11:36:59 +02:00
|
|
|
|
2019-10-10 11:33:51 +02:00
|
|
|
@Test
|
|
|
|
public void testTransformSaxonHE() throws Exception {
|
2019-10-24 11:36:59 +02:00
|
|
|
|
|
|
|
Map<String, Vocabulary> vocabularies = new HashMap<>();
|
|
|
|
vocabularies.put("dnet:languages", VocabularyHelper.getVocabularyFromAPI("dnet:languages"));
|
|
|
|
Cleaner cleanFunction = new Cleaner(vocabularies);
|
2019-10-10 11:33:51 +02:00
|
|
|
Processor proc = new Processor(false);
|
|
|
|
proc.registerExtensionFunction(cleanFunction);
|
|
|
|
final XsltCompiler comp = proc.newXsltCompiler();
|
|
|
|
XsltExecutable exp = comp.compile(new StreamSource(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/ext_simple.xsl")));
|
|
|
|
XdmNode source = proc.newDocumentBuilder().build(new StreamSource(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input.xml")));
|
|
|
|
XsltTransformer trans = exp.load();
|
|
|
|
trans.setInitialContextNode(source);
|
|
|
|
final StringWriter output = new StringWriter();
|
|
|
|
Serializer out = proc.newSerializer(output);
|
|
|
|
out.setOutputProperty(Serializer.Property.METHOD,"xml");
|
|
|
|
out.setOutputProperty(Serializer.Property.INDENT, "yes");
|
|
|
|
trans.setDestination(out);
|
|
|
|
trans.transform();
|
|
|
|
System.out.println(output.toString());
|
|
|
|
}
|
|
|
|
|
2020-03-25 17:59:45 +01:00
|
|
|
@DisplayName("Test TransformSparkJobNode.main")
|
2019-04-11 15:39:29 +02:00
|
|
|
@Test
|
2020-03-25 17:59:45 +01:00
|
|
|
public void transformTest(@TempDir Path testDir) throws Exception {
|
2019-10-24 11:36:59 +02:00
|
|
|
final String mdstore_input = this.getClass().getResource("/eu/dnetlib/dhp/transform/mdstorenative").getFile();
|
|
|
|
final String mdstore_output = testDir.toString()+"/version";
|
2019-04-11 15:39:29 +02:00
|
|
|
final String xslt = DHPUtils.compressString(IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/tr.xml")));
|
2019-10-24 11:36:59 +02:00
|
|
|
TransformSparkJobNode.main(new String[]{
|
|
|
|
"-mt", "local",
|
|
|
|
"-i", mdstore_input,
|
|
|
|
"-o", mdstore_output,
|
|
|
|
"-d", "1",
|
|
|
|
"-w", "1",
|
|
|
|
"-tr", xslt,
|
|
|
|
"-t", "true",
|
|
|
|
"-ru", "",
|
|
|
|
"-rp", "",
|
|
|
|
"-rh", "",
|
|
|
|
"-ro", "",
|
|
|
|
"-rr", ""});
|
2019-04-11 15:39:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void tryLoadFolderOnCP() throws Exception {
|
2019-10-24 11:36:59 +02:00
|
|
|
final String path = this.getClass().getResource("/eu/dnetlib/dhp/transform/mdstorenative").getFile();
|
2019-04-11 15:39:29 +02:00
|
|
|
System.out.println("path = " + path);
|
|
|
|
|
2019-12-06 13:38:00 +01:00
|
|
|
Path tempDirWithPrefix = Files.createTempDirectory("mdstore_output");
|
2019-04-11 15:39:29 +02:00
|
|
|
|
|
|
|
System.out.println(tempDirWithPrefix.toFile().getAbsolutePath());
|
|
|
|
|
|
|
|
Files.deleteIfExists(tempDirWithPrefix);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testTransformFunction() throws Exception {
|
|
|
|
SAXReader reader = new SAXReader();
|
|
|
|
Document document = reader.read(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/tr.xml"));
|
|
|
|
Node node = document.selectSingleNode("//CODE/*[local-name()='stylesheet']");
|
|
|
|
final String xslt = node.asXML();
|
2019-10-24 11:36:59 +02:00
|
|
|
Map<String, Vocabulary> vocabularies = new HashMap<>();
|
|
|
|
vocabularies.put("dnet:languages", VocabularyHelper.getVocabularyFromAPI("dnet:languages"));
|
2019-04-11 15:39:29 +02:00
|
|
|
|
2019-10-24 11:36:59 +02:00
|
|
|
TransformFunction tf = new TransformFunction(accumulator, accumulator, accumulator, xslt, 1, vocabularies);
|
2019-04-11 15:39:29 +02:00
|
|
|
|
|
|
|
MetadataRecord record = new MetadataRecord();
|
|
|
|
record.setBody(IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/input.xml")));
|
|
|
|
|
|
|
|
final MetadataRecord result = tf.call(record);
|
2020-03-25 17:59:45 +01:00
|
|
|
assertNotNull(result.getBody());
|
2019-10-10 11:33:51 +02:00
|
|
|
|
|
|
|
System.out.println(result.getBody());
|
2019-04-11 15:39:29 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void extractTr() throws Exception {
|
|
|
|
|
|
|
|
final String xmlTr = IOUtils.toString(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/tr.xml"));
|
|
|
|
|
|
|
|
SAXReader reader = new SAXReader();
|
|
|
|
Document document = reader.read(this.getClass().getResourceAsStream("/eu/dnetlib/dhp/transform/tr.xml"));
|
|
|
|
Node node = document.selectSingleNode("//CODE/*[local-name()='stylesheet']");
|
|
|
|
|
|
|
|
System.out.println(node.asXML());
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|