andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[bip affiliations] considers only DOI based records

This commit is contained in:
Claudio Atzori 2024-08-05 12:13:48 +02:00
parent fecbf93e0e
commit 8e7ef79ce0
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java
View File

@ -129,7 +129,8 @@ public class PrepareAffiliationRelations implements Serializable {
Dataset<Row> df = spark
.read()
.schema("`DOI` STRING, `Matchings` ARRAY<STRUCT<`RORid`:STRING,`Confidence`:DOUBLE>>")
.json(inputPath);
.json(inputPath)
.where("DOI is not null");
// unroll nested arrays
df = df