mkallipo
/
affRo
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Oalex #13

Merged
mkallipo merged 15 commits from openaire-workflow-ready_2 into openaire-workflow-ready 2024-12-09 18:51:24 +01:00
Conversation 0 Commits 15 Files Changed 1 +3 -2
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit e33bf4ef14 - Show all commits

5
strings.py
View File

@ -30,10 +30,11 @@ spark.read.json(folder_path)
explode(col("raw_aff_string")).alias("aff_string") #this allows to split all the raw_aff_string and to parallelize better explode(col("raw_aff_string")).alias("aff_string") #this allows to split all the raw_aff_string and to parallelize better
) )
.withColumn("Matchings", oalex_affro(col("doi"), col("aff_string"))) #this one says create a new column with name Matchinds as the result of the function as second argument .withColumn("Matchings", oalex_affro(col("doi"), col("aff_string"))) #this one says create a new column with name Matchinds as the result of the function as second argument
.select("DOI","OAlex","Matchins") .drop(col("aff_string")
.select(col("DOI"),col("OAlex"),explode("Matchins").alias("match")
.groupBy("DOI") #this groups by doi to have just one row per each doi .groupBy("DOI") #this groups by doi to have just one row per each doi
.agg(first("OAlex").alias("OAlex"), #for each DOI it says what are the other columns Since OALEX is equal for each doi just select the first, while use the collect_list function to aggregate the Matchings .agg(first("OAlex").alias("OAlex"), #for each DOI it says what are the other columns Since OALEX is equal for each doi just select the first, while use the collect_list function to aggregate the Matchings
collect_list("Matchings").alias("Matchings")) collect_list("match").alias("Matchings"))
.write .write
.mode("overwrite") #in case the folder already exists on hadoop it does not break .mode("overwrite") #in case the folder already exists on hadoop it does not break
.option("compression","gzip") #to reduce the space .option("compression","gzip") #to reduce the space