D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

aligned with the last version of pom for production

This commit is contained in:
Miriam Baglioni 2023-06-02 16:13:18 +02:00
parent 2e8639f22d
commit 8661bc0c90
7 changed files with 49 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dump-schema/src/test/java/GenerateJsonSchema.java
View File

@ -1,5 +1,6 @@
import java.io.IOException;
import eu.dnetlib.dhp.oa.model.graph.*;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -10,10 +11,6 @@ import com.github.victools.jsonschema.generator.*;
import eu.dnetlib.dhp.ExecCreateSchemas;
import eu.dnetlib.dhp.oa.model.community.CommunityResult;
import eu.dnetlib.dhp.oa.model.graph.Datasource;
import eu.dnetlib.dhp.oa.model.graph.GraphResult;
import eu.dnetlib.dhp.oa.model.graph.Organization;
import eu.dnetlib.dhp.oa.model.graph.ResearchCommunity;
//@Disabled
class GenerateJsonSchema {
@ -44,7 +41,7 @@ class GenerateJsonSchema {
.without(Option.NONPUBLIC_NONSTATIC_FIELDS_WITHOUT_GETTERS);
SchemaGeneratorConfig config = configBuilder.build();
SchemaGenerator generator = new SchemaGenerator(config);
JsonNode jsonSchema = generator.generateSchema(GraphResult.class);
JsonNode jsonSchema = generator.generateSchema(Project.class);
System.out.println(jsonSchema.toString());
}

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/Extractor.java
View File

@ -15,7 +15,6 @@ import org.apache.spark.sql.SparkSession;
import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.model.Provenance;
import eu.dnetlib.dhp.oa.model.graph.RelType;
import eu.dnetlib.dhp.oa.model.graph.Relation;
import eu.dnetlib.dhp.schema.common.ModelConstants;

5
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpEntitiesJob.java
View File

@ -337,8 +337,9 @@ public class SparkDumpEntitiesJob implements Serializable {
.ofNullable(d.getJournal())
.ifPresent(j -> datasource.setJournal(getContainer(j)));
Optional.ofNullable(d.getMeasures())
.ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures())));
Optional
.ofNullable(d.getMeasures())
.ifPresent(m -> datasource.setIndicators(Utils.getIndicator(d.getMeasures())));
return datasource;

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkDumpRelationJob.java
View File

@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.oa.model.Provenance;
import eu.dnetlib.dhp.oa.model.graph.RelType;
import eu.dnetlib.dhp.schema.common.ModelSupport;
import eu.dnetlib.dhp.schema.oaf.DataInfo;

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/complete/SparkOrganizationRelation.java
View File

@ -24,7 +24,6 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.oa.graph.dump.Utils;
import eu.dnetlib.dhp.oa.graph.dump.community.CommunityMap;
import eu.dnetlib.dhp.oa.model.Provenance;
import eu.dnetlib.dhp.oa.model.graph.RelType;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import eu.dnetlib.dhp.schema.common.ModelSupport;

62
dump/src/test/java/eu/dnetlib/dhp/oa/graph/dump/complete/DumpOrganizationProjectDatasourceTest.java
View File

@ -6,7 +6,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import eu.dnetlib.dhp.oa.model.Indicator;
import org.apache.commons.io.FileUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
@ -25,6 +24,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.oa.graph.dump.exceptions.NoAvailableEntityTypeException;
import eu.dnetlib.dhp.oa.model.Indicator;
import eu.dnetlib.dhp.schema.oaf.Datasource;
import eu.dnetlib.dhp.schema.oaf.Organization;
import eu.dnetlib.dhp.schema.oaf.Project;
@ -99,7 +99,7 @@ public class DumpOrganizationProjectDatasourceTest {
Assertions.assertEquals(15, verificationDataset.count());
//TODO write significant assertions
// TODO write significant assertions
// verificationDataset
// .foreach(
// (ForeachFunction<eu.dnetlib.dhp.oa.model.graph.Organization>) o -> System.out
@ -137,19 +137,31 @@ public class DumpOrganizationProjectDatasourceTest {
Assertions.assertEquals(10, verificationDataset.filter("indicators is NULL").count());
Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count());
Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::01bb7b48e29d732a1c7bc5150b9195c4'").count());
Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'").count());
Assertions
.assertEquals(
1,
verificationDataset
.filter("indicators is not NULL AND id == 'aka_________::01bb7b48e29d732a1c7bc5150b9195c4'")
.count());
Assertions
.assertEquals(
1,
verificationDataset
.filter("indicators is not NULL AND id == 'aka_________::9d1af21dbd0f5bc719f71553d19a6b3a'")
.count());
eu.dnetlib.dhp.oa.model.graph.Project p = tmp.filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4")).first();
Assertions.assertEquals("2019",p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("1804",p.getIndicators().getUsageCounts().getViews());
eu.dnetlib.dhp.oa.model.graph.Project p = tmp
.filter(pr -> pr.getId().equals("aka_________::01bb7b48e29d732a1c7bc5150b9195c4"))
.first();
Assertions.assertEquals("2019", p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("1804", p.getIndicators().getUsageCounts().getViews());
Assertions.assertNull(p.getIndicators().getImpactMeasures());
p = tmp.filter(pr -> pr.getId().equals("aka_________::9d1af21dbd0f5bc719f71553d19a6b3a")).first();
Assertions.assertEquals("139",p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("53",p.getIndicators().getUsageCounts().getViews());
Assertions.assertEquals("139", p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("53", p.getIndicators().getUsageCounts().getViews());
Assertions.assertNull(p.getIndicators().getImpactMeasures());
//TODO write significant assertions
// TODO write significant assertions
// verificationDataset
// .foreach(
// (ForeachFunction<eu.dnetlib.dhp.oa.model.graph.Project>) o -> System.out
@ -186,20 +198,32 @@ public class DumpOrganizationProjectDatasourceTest {
Assertions.assertEquals(3, verificationDataset.filter("indicators is NULL").count());
Assertions.assertEquals(2, verificationDataset.filter("indicators is not NULL").count());
Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::1fa6859d71faa77b32d82f278c6ed1df'").count());
Assertions.assertEquals(1, verificationDataset.filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'").count());
Assertions
.assertEquals(
1,
verificationDataset
.filter("indicators is not NULL AND id == 'doajarticles::1fa6859d71faa77b32d82f278c6ed1df'")
.count());
Assertions
.assertEquals(
1,
verificationDataset
.filter("indicators is not NULL AND id == 'doajarticles::9c4b678901e5276d9e3addee566816af'")
.count());
eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp.filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df")).first();
Assertions.assertEquals("47542",p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("36485",p.getIndicators().getUsageCounts().getViews());
eu.dnetlib.dhp.oa.model.graph.Datasource p = tmp
.filter(pr -> pr.getId().equals("doajarticles::1fa6859d71faa77b32d82f278c6ed1df"))
.first();
Assertions.assertEquals("47542", p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("36485", p.getIndicators().getUsageCounts().getViews());
Assertions.assertNull(p.getIndicators().getImpactMeasures());
p = tmp.filter(pr -> pr.getId().equals("doajarticles::9c4b678901e5276d9e3addee566816af")).first();
Assertions.assertEquals("981357",p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("646539",p.getIndicators().getUsageCounts().getViews());
Assertions.assertEquals("981357", p.getIndicators().getUsageCounts().getDownloads());
Assertions.assertEquals("646539", p.getIndicators().getUsageCounts().getViews());
Assertions.assertNull(p.getIndicators().getImpactMeasures());
//TODO write significant assertions
// TODO write significant assertions
// verificationDataset
// .foreach(
// (ForeachFunction<eu.dnetlib.dhp.oa.model.graph.Datasource>) o -> System.out
@ -233,7 +257,7 @@ public class DumpOrganizationProjectDatasourceTest {
Assertions.assertEquals(1, verificationDataset.count());
//TODO write significant assertions
// TODO write significant assertions
// verificationDataset
// .foreach(
// (ForeachFunction<eu.dnetlib.dhp.oa.model.graph.Datasource>) o -> System.out

2
pom.xml
View File

@ -102,7 +102,7 @@
<junit-jupiter.version>5.6.1</junit-jupiter.version>
<dhp.commons.lang.version>3.5</dhp.commons.lang.version>
<dhp.guava.version>11.0.2</dhp.guava.version>
<dhp-schemas.version>[3.16.0]</dhp-schemas.version>
<dhp-schemas.version>[2.13.1-patched]</dhp-schemas.version>
</properties>
</project>