package eu.dnetlib.dhp.swh; import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; public class PrepareSWHActionsetsTest { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static SparkSession spark; private static Path workingDir; private static final Logger log = LoggerFactory .getLogger(PrepareSWHActionsetsTest.class); @BeforeAll public static void beforeAll() throws IOException { workingDir = Files.createTempDirectory(PrepareSWHActionsetsTest.class.getSimpleName()); log.info("Using work dir {}", workingDir); SparkConf conf = new SparkConf(); conf.setAppName(PrepareSWHActionsetsTest.class.getSimpleName()); conf.setMaster("local[*]"); conf.set("spark.driver.host", "localhost"); conf.set("hive.metastore.local", "true"); conf.set("spark.ui.enabled", "false"); conf.set("spark.sql.warehouse.dir", workingDir.toString()); conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); spark = SparkSession .builder() .appName(PrepareSWHActionsetsTest.class.getSimpleName()) .config(conf) .getOrCreate(); } @AfterAll public static void afterAll() throws IOException { FileUtils.deleteDirectory(workingDir.toFile()); spark.stop(); } @Test void testRun() throws Exception { String lastVisitsPath = getClass() .getResource("/eu/dnetlib/dhp/swh/last_visits_data.seq") .getPath(); String outputPath = workingDir.toString() + "/actionSet"; String softwareInputPath = getClass() .getResource("/eu/dnetlib/dhp/swh/software.json.gz") .getPath(); PrepareSWHActionsets .main( new String[] { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-lastVisitsPath", lastVisitsPath, "-softwareInputPath", softwareInputPath, "-actionsetsPath", outputPath }); } }