1
0
Fork 0

minor changes and refactoring

This commit is contained in:
Miriam Baglioni 2021-07-13 17:10:02 +02:00
parent 59615da65e
commit 618d2de2da
8 changed files with 176 additions and 175 deletions

View File

@ -428,7 +428,7 @@ public class ResultMapper implements Serializable {
if (oPca.isPresent() && oPcc.isPresent()) { if (oPca.isPresent() && oPcc.isPresent()) {
Field<String> pca = oPca.get(); Field<String> pca = oPca.get();
Field<String> pcc = oPcc.get(); Field<String> pcc = oPcc.get();
if(!pca.getValue().trim().equals("") && !pcc.getValue().trim().equals("")){ if (!pca.getValue().trim().equals("") && !pcc.getValue().trim().equals("")) {
APC apc = new APC(); APC apc = new APC();
apc.setCurrency(oPcc.get().getValue()); apc.setCurrency(oPcc.get().getValue());
apc.setAmount(oPca.get().getValue()); apc.setAmount(oPca.get().getValue());

View File

@ -70,10 +70,10 @@ public class CreateContextRelation implements Serializable {
cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class)); cce.execute(Process::getRelation, CONTEX_RELATION_DATASOURCE, ModelSupport.getIdPrefix(Datasource.class));
log.info("Creating relations for projects... "); log.info("Creating relations for projects... ");
// cce cce
// .execute( .execute(
// Process::getRelation, CONTEX_RELATION_PROJECT, Process::getRelation, CONTEX_RELATION_PROJECT,
// ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class)); ModelSupport.getIdPrefix(eu.dnetlib.dhp.schema.oaf.Project.class));
cce.close(); cce.close();

View File

@ -461,7 +461,7 @@ public class DumpGraphEntities implements Serializable {
} }
private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) { private static Organization mapOrganization(eu.dnetlib.dhp.schema.oaf.Organization org) {
if(org.getDataInfo().getDeletedbyinference()) if (org.getDataInfo().getDeletedbyinference())
return null; return null;
Organization organization = new Organization(); Organization organization = new Organization();

View File

@ -147,7 +147,7 @@ public class Extractor implements Serializable {
.map( .map(
paction -> Provenance paction -> Provenance
.newInstance( .newInstance(
paction.getClassid(), paction.getClassname(),
dinfo.getTrust())) dinfo.getTrust()))
.orElse( .orElse(
Provenance Provenance

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.oa.graph.dump.complete; package eu.dnetlib.dhp.oa.graph.dump.complete;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
@ -25,7 +26,6 @@ import eu.dnetlib.dhp.schema.oaf.*;
* with this view for both the source and the target * with this view for both the source and the target
*/ */
public class SparkSelectValidRelationsJob implements Serializable { public class SparkSelectValidRelationsJob implements Serializable {
private static final Logger log = LoggerFactory.getLogger(SparkSelectValidRelationsJob.class); private static final Logger log = LoggerFactory.getLogger(SparkSelectValidRelationsJob.class);
@ -114,7 +114,7 @@ public class SparkSelectValidRelationsJob implements Serializable {
"UNION ALL " + "UNION ALL " +
"SELECT id " + "SELECT id " +
"FROM datasource " + "FROM datasource " +
"WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false " ) "WHERE datainfo.deletedbyinference = false AND datainfo.invisible = false ")
.createOrReplaceTempView("identifiers"); .createOrReplaceTempView("identifiers");
spark spark

View File

@ -436,13 +436,15 @@ public class DumpJobTest {
.createDataset(tmp.rdd(), Encoders.bean(GraphResult.class)); .createDataset(tmp.rdd(), Encoders.bean(GraphResult.class));
Assertions.assertEquals(23, verificationDataset.count()); Assertions.assertEquals(23, verificationDataset.count());
//verificationDataset.show(false); // verificationDataset.show(false);
Assertions.assertEquals(23, verificationDataset.filter("type = 'publication'").count()); Assertions.assertEquals(23, verificationDataset.filter("type = 'publication'").count());
verificationDataset.createOrReplaceTempView("check"); verificationDataset.createOrReplaceTempView("check");
org.apache.spark.sql.Dataset<Row> temp = spark.sql("select id " + org.apache.spark.sql.Dataset<Row> temp = spark
.sql(
"select id " +
"from check " + "from check " +
"lateral view explode (instance) i as inst " + "lateral view explode (instance) i as inst " +
"where inst.articleprocessingcharge is not null"); "where inst.articleprocessingcharge is not null");
@ -453,8 +455,6 @@ public class DumpJobTest {
Assertions.assertTrue(temp.filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'").count() == 1); Assertions.assertTrue(temp.filter("id = '50|dedup_wf_001::01e6a28565ca01376b7548e530c6f6e8'").count() == 1);
// verificationDataset.filter("bestAccessright.code = 'c_abf2'").count() == verificationDataset // verificationDataset.filter("bestAccessright.code = 'c_abf2'").count() == verificationDataset
// .filter("bestAccessright.code = 'c_abf2' and bestAccessright.label = 'OPEN'") // .filter("bestAccessright.code = 'c_abf2' and bestAccessright.label = 'OPEN'")
// .count() // .count()

View File

@ -97,7 +97,7 @@ public class CreateEntityTest {
Assertions.assertEquals(12, riList.size()); Assertions.assertEquals(12, riList.size());
riList.stream().forEach(c -> { riList.stream().forEach(c -> {
switch (c.getOriginalId()) { switch (c.getAcronym()) {
case "mes": case "mes":
Assertions Assertions
.assertTrue(c.getType().equals(eu.dnetlib.dhp.oa.graph.dump.Constants.RESEARCH_COMMUNITY)); .assertTrue(c.getType().equals(eu.dnetlib.dhp.oa.graph.dump.Constants.RESEARCH_COMMUNITY));
@ -115,9 +115,9 @@ public class CreateEntityTest {
String String
.format( .format(
"%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX,
DHPUtils.md5(c.getOriginalId())))); DHPUtils.md5(c.getAcronym()))));
Assertions.assertTrue(c.getZenodo_community().equals("https://zenodo.org/communities/oac_mes")); Assertions.assertTrue(c.getZenodo_community().equals("https://zenodo.org/communities/oac_mes"));
Assertions.assertTrue("mes".equals(c.getOriginalId())); Assertions.assertTrue("mes".equals(c.getAcronym()));
break; break;
case "clarin": case "clarin":
Assertions Assertions
@ -130,9 +130,9 @@ public class CreateEntityTest {
String String
.format( .format(
"%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX, "%s|%s::%s", Constants.CONTEXT_ID, Constants.CONTEXT_NS_PREFIX,
DHPUtils.md5(c.getOriginalId())))); DHPUtils.md5(c.getAcronym()))));
Assertions.assertTrue(c.getZenodo_community().equals("https://zenodo.org/communities/oac_clarin")); Assertions.assertTrue(c.getZenodo_community().equals("https://zenodo.org/communities/oac_clarin"));
Assertions.assertTrue("clarin".equals(c.getOriginalId())); Assertions.assertTrue("clarin".equals(c.getAcronym()));
break; break;
} }
// TODO add check for all the others Entities // TODO add check for all the others Entities

View File

@ -1,3 +1,4 @@
package eu.dnetlib.dhp.oa.graph.dump.complete; package eu.dnetlib.dhp.oa.graph.dump.complete;
import java.io.IOException; import java.io.IOException;