D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 10 Projects 2 Releases 1 Wiki Activity

master #59

Closed
claudio.atzori wants to merge 3221 commits from master into stable_ids
pull from: master
merge into: D-Net:stable_ids
Conversation 0 Commits 3221 Files Changed 2141 +50 -46
4 changed files with 50 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7aebedb43c - Show all commits

1
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java
View File

@ -9,7 +9,6 @@ import java.io.Serializable;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.sql.*; import org.apache.spark.sql.*;

9
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java
View File

@ -83,9 +83,12 @@ public class SparkAtomicActionUsageJob implements Serializable {
private static void prepareData(String dbname, SparkSession spark, String workingPath, String tableName, private static void prepareData(String dbname, SparkSession spark, String workingPath, String tableName,
String attribute_name) { String attribute_name) {
spark spark
.sql(String.format( .sql(
"select %s as id, sum(downloads) as downloads, sum(views) as views " + String
"from %s.%s group by %s", attribute_name, dbname, tableName, attribute_name)) .format(
"select %s as id, sum(downloads) as downloads, sum(views) as views " +
"from %s.%s group by %s",
attribute_name, dbname, tableName, attribute_name))
.as(Encoders.bean(UsageStatsModel.class)) .as(Encoders.bean(UsageStatsModel.class))
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)

81
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java
View File

@ -1,7 +1,10 @@
package eu.dnetlib.dhp.actionmanager.createunresolvedentities; package eu.dnetlib.dhp.actionmanager.createunresolvedentities;
import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException;
import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel; import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -17,9 +20,9 @@ import org.junit.jupiter.api.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.file.Files;
import java.nio.file.Path; import eu.dnetlib.dhp.actionmanager.createunresolvedentities.model.FOSDataModel;
/** /**
* @author miriam.baglioni * @author miriam.baglioni
@ -27,48 +30,48 @@ import java.nio.file.Path;
*/ */
public class GetFosTest { public class GetFosTest {
private static final Logger log = LoggerFactory.getLogger(ProduceTest.class); private static final Logger log = LoggerFactory.getLogger(ProduceTest.class);
private static Path workingDir; private static Path workingDir;
private static SparkSession spark; private static SparkSession spark;
private static LocalFileSystem fs; private static LocalFileSystem fs;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@BeforeAll @BeforeAll
public static void beforeAll() throws IOException { public static void beforeAll() throws IOException {
workingDir = Files.createTempDirectory(PrepareTest.class.getSimpleName()); workingDir = Files.createTempDirectory(PrepareTest.class.getSimpleName());
fs = FileSystem.getLocal(new Configuration()); fs = FileSystem.getLocal(new Configuration());
log.info("using work dir {}", workingDir); log.info("using work dir {}", workingDir);
SparkConf conf = new SparkConf(); SparkConf conf = new SparkConf();
conf.setAppName(ProduceTest.class.getSimpleName()); conf.setAppName(ProduceTest.class.getSimpleName());
conf.setMaster("local[*]"); conf.setMaster("local[*]");
conf.set("spark.driver.host", "localhost"); conf.set("spark.driver.host", "localhost");
conf.set("hive.metastore.local", "true"); conf.set("hive.metastore.local", "true");
conf.set("spark.ui.enabled", "false"); conf.set("spark.ui.enabled", "false");
conf.set("spark.sql.warehouse.dir", workingDir.toString()); conf.set("spark.sql.warehouse.dir", workingDir.toString());
conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString());
spark = SparkSession spark = SparkSession
.builder() .builder()
.appName(PrepareTest.class.getSimpleName()) .appName(PrepareTest.class.getSimpleName())
.config(conf) .config(conf)
.getOrCreate(); .getOrCreate();
} }
@AfterAll @AfterAll
public static void afterAll() throws IOException { public static void afterAll() throws IOException {
FileUtils.deleteDirectory(workingDir.toFile()); FileUtils.deleteDirectory(workingDir.toFile());
spark.stop(); spark.stop();
} }
@Test
@Test
void test3() throws Exception { void test3() throws Exception {
final String sourcePath = getClass() final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs.tsv") .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs.tsv")
.getPath(); .getPath();
final String outputPath = workingDir.toString() + "/fos.json"; final String outputPath = workingDir.toString() + "/fos.json";
GetFOSSparkJob GetFOSSparkJob

5
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/SolrConfigTest.java
View File

@ -115,9 +115,8 @@ public class SolrConfigTest extends SolrTest {
for (SolrDocument doc : rsp.getResults()) { for (SolrDocument doc : rsp.getResults()) {
System.out System.out
.println( .println(
doc.get("__indexrecordidentifier") + "\t" + doc.get("__indexrecordidentifier") + "\t" +
doc.get("__result") + "\t" doc.get("__result") + "\t");
);
} }
} }
} }