|
|
|
@ -5,7 +5,6 @@ import static eu.dnetlib.dhp.common.Constants.MDSTORE_DATA_PATH;
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.nio.file.Files;
|
|
|
|
|
import java.nio.file.Path;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
@ -35,26 +34,11 @@ import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
|
|
|
|
@ExtendWith(MockitoExtension.class)
|
|
|
|
|
public class TransformationJobTest extends AbstractVocabularyTest {
|
|
|
|
|
|
|
|
|
|
private static SparkSession spark;
|
|
|
|
|
|
|
|
|
|
@BeforeAll
|
|
|
|
|
public static void beforeAll() throws IOException, ISLookUpException {
|
|
|
|
|
SparkConf conf = new SparkConf();
|
|
|
|
|
conf.setAppName(TransformationJobTest.class.getSimpleName());
|
|
|
|
|
conf.setMaster("local");
|
|
|
|
|
spark = SparkSession.builder().config(conf).getOrCreate();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@BeforeEach
|
|
|
|
|
public void setUp() throws IOException, ISLookUpException {
|
|
|
|
|
setUpVocabulary();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@AfterAll
|
|
|
|
|
public static void afterAll() {
|
|
|
|
|
spark.stop();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
@DisplayName("Test Date cleaner")
|
|
|
|
|
public void testDateCleaner() throws Exception {
|
|
|
|
@ -82,68 +66,61 @@ public class TransformationJobTest extends AbstractVocabularyTest {
|
|
|
|
|
// Print the record
|
|
|
|
|
System.out.println(result.getBody());
|
|
|
|
|
// TODO Create significant Assert
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@DisplayName("Test TransformSparkJobNode.main")
|
|
|
|
|
@Test
|
|
|
|
|
@DisplayName("Test TransformSparkJobNode.main")
|
|
|
|
|
public void transformTest(@TempDir Path testDir) throws Exception {
|
|
|
|
|
|
|
|
|
|
final String mdstore_input = this.getClass().getResource("/eu/dnetlib/dhp/transform/mdstorenative").getFile();
|
|
|
|
|
final String mdstore_output = testDir.toString() + "/version";
|
|
|
|
|
|
|
|
|
|
mockupTrasformationRule("simpleTRule", "/eu/dnetlib/dhp/transform/ext_simple.xsl");
|
|
|
|
|
|
|
|
|
|
final Map<String, String> parameters = Stream.of(new String[][] {
|
|
|
|
|
{
|
|
|
|
|
"dateOfTransformation", "1234"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"transformationPlugin", "XSLT_TRANSFORM"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"transformationRuleId", "simpleTRule"
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
}).collect(Collectors.toMap(data -> data[0], data -> data[1]));
|
|
|
|
|
SparkConf conf = new SparkConf();
|
|
|
|
|
conf.setAppName(TransformationJobTest.class.getSimpleName());
|
|
|
|
|
conf.setMaster("local");
|
|
|
|
|
|
|
|
|
|
TransformSparkJobNode.transformRecords(parameters, isLookUpService, spark, mdstore_input, mdstore_output);
|
|
|
|
|
try(SparkSession spark = SparkSession.builder().config(conf).getOrCreate()) {
|
|
|
|
|
|
|
|
|
|
// TODO introduce useful assertions
|
|
|
|
|
final String mdstore_input = this.getClass().getResource("/eu/dnetlib/dhp/transform/mdstorenative").getFile();
|
|
|
|
|
final String mdstore_output = testDir.toString() + "/version";
|
|
|
|
|
|
|
|
|
|
final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
|
|
|
|
|
final Dataset<MetadataRecord> mOutput = spark
|
|
|
|
|
.read()
|
|
|
|
|
.format("parquet")
|
|
|
|
|
.load(mdstore_output + MDSTORE_DATA_PATH)
|
|
|
|
|
.as(encoder);
|
|
|
|
|
mockupTrasformationRule("simpleTRule", "/eu/dnetlib/dhp/transform/ext_simple.xsl");
|
|
|
|
|
|
|
|
|
|
final Long total = mOutput.count();
|
|
|
|
|
final Map<String, String> parameters = Stream.of(new String[][]{
|
|
|
|
|
{
|
|
|
|
|
"dateOfTransformation", "1234"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"transformationPlugin", "XSLT_TRANSFORM"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"transformationRuleId", "simpleTRule"
|
|
|
|
|
},
|
|
|
|
|
|
|
|
|
|
final long recordTs = mOutput
|
|
|
|
|
.filter((FilterFunction<MetadataRecord>) p -> p.getDateOfTransformation() == 1234)
|
|
|
|
|
.count();
|
|
|
|
|
}).collect(Collectors.toMap(data -> data[0], data -> data[1]));
|
|
|
|
|
|
|
|
|
|
final long recordNotEmpty = mOutput
|
|
|
|
|
.filter((FilterFunction<MetadataRecord>) p -> !StringUtils.isBlank(p.getBody()))
|
|
|
|
|
.count();
|
|
|
|
|
TransformSparkJobNode.transformRecords(parameters, isLookUpService, spark, mdstore_input, mdstore_output);
|
|
|
|
|
|
|
|
|
|
assertEquals(total, recordTs);
|
|
|
|
|
// TODO introduce useful assertions
|
|
|
|
|
|
|
|
|
|
assertEquals(total, recordNotEmpty);
|
|
|
|
|
final Encoder<MetadataRecord> encoder = Encoders.bean(MetadataRecord.class);
|
|
|
|
|
final Dataset<MetadataRecord> mOutput = spark
|
|
|
|
|
.read()
|
|
|
|
|
.format("parquet")
|
|
|
|
|
.load(mdstore_output + MDSTORE_DATA_PATH)
|
|
|
|
|
.as(encoder);
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
final Long total = mOutput.count();
|
|
|
|
|
|
|
|
|
|
@Test
|
|
|
|
|
public void tryLoadFolderOnCP() throws Exception {
|
|
|
|
|
final String path = this.getClass().getResource("/eu/dnetlib/dhp/transform/mdstorenative").getFile();
|
|
|
|
|
System.out.println("path = " + path);
|
|
|
|
|
final long recordTs = mOutput
|
|
|
|
|
.filter((FilterFunction<MetadataRecord>) p -> p.getDateOfTransformation() == 1234)
|
|
|
|
|
.count();
|
|
|
|
|
|
|
|
|
|
Path tempDirWithPrefix = Files.createTempDirectory("mdstore_output");
|
|
|
|
|
final long recordNotEmpty = mOutput
|
|
|
|
|
.filter((FilterFunction<MetadataRecord>) p -> !StringUtils.isBlank(p.getBody()))
|
|
|
|
|
.count();
|
|
|
|
|
|
|
|
|
|
System.out.println(tempDirWithPrefix.toFile().getAbsolutePath());
|
|
|
|
|
assertEquals(total, recordTs);
|
|
|
|
|
|
|
|
|
|
Files.deleteIfExists(tempDirWithPrefix);
|
|
|
|
|
assertEquals(total, recordNotEmpty);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private XSLTTransformationFunction loadTransformationRule(final String path) throws Exception {
|
|
|
|
|