mkallipo
/
affRo
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

[oalex] change collec_list to collect_set so that the same match will be there just one time

This commit is contained in:
Miriam Baglioni 2024-12-05 21:26:08 +01:00
parent 0043e4051f
commit 2806511e02
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
strings.py
View File

@ -5,7 +5,7 @@ from pyspark.sql.types import StringType, ArrayType, StructType, StructField, Do
from affro_cluster import *
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, explode, first, collect_list, udf
from pyspark.sql.functions import col, explode, first, collect_set, udf
import sys
spark = SparkSession.builder.appName("JSONProcessing").getOrCreate()
@ -81,7 +81,7 @@ affs.join(exploded, on="affiliation") \
) \
.groupBy("DOI") \
.agg(first("OAlex").alias("OAlex"), #for each DOI it says what are the other columns Since OALEX is equal for each doi just select the first, while use the collect_list function to aggregate the Matchings
collect_list("match").alias("Matchings") #each exploded match is collected again
collect_set("match").alias("Matchings") #each exploded match is collected again
) \
.write \
.mode("overwrite") \