From 1c82acb16835ea7ce3f70f8c847204f0ac487708 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 25 Jul 2022 14:26:39 +0200 Subject: [PATCH] [EOSC Context Tagging] refactoring: moved EOSC IF tagging in package eosc under bulkTag --- .../eu/dnetlib/dhp/bulktag/{ => eosc}/SparkEoscTag.java | 7 +------ .../eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml | 2 +- .../test/java/eu/dnetlib/dhp/bulktag/EOSCTagJobTest.java | 5 +---- 3 files changed, 3 insertions(+), 11 deletions(-) rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/{ => eosc}/SparkEoscTag.java (96%) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkEoscTag.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/eosc/SparkEoscTag.java similarity index 96% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkEoscTag.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/eosc/SparkEoscTag.java index 730e8a3fe..736496641 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkEoscTag.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/eosc/SparkEoscTag.java @@ -1,13 +1,10 @@ -package eu.dnetlib.dhp.bulktag; +package eu.dnetlib.dhp.bulktag.eosc; import static eu.dnetlib.dhp.PropagationConstant.readPath; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; @@ -21,9 +18,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; public class SparkEoscTag { private static final Logger log = LoggerFactory.getLogger(SparkEoscTag.class); diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index 17fc76aad..9c1bbdf72 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -226,7 +226,7 @@ yarn-cluster cluster EOSC_tagging - eu.dnetlib.dhp.bulktag.SparkEoscTag + eu.dnetlib.dhp.bulktag.eosc.SparkEoscTag dhp-enrichment-${projectVersion}.jar --num-executors=${sparkExecutorNumber} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/EOSCTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/EOSCTagJobTest.java index 5f47da10e..17e570fdf 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/EOSCTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/EOSCTagJobTest.java @@ -1,21 +1,18 @@ package eu.dnetlib.dhp.bulktag; -import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY_INDICATOR; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import eu.dnetlib.dhp.bulktag.eosc.SparkEoscTag; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll;