D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +20 -4
3 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ee1fcb672b - Show all commits

2
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/crossref/Crossref2Oaf.scala
View File

@ -249,7 +249,7 @@ case object Crossref2Oaf {
val subtitles = val subtitles =
for { JString(title) <- json \ "subtitle" if title.nonEmpty } yield structuredProperty( for { JString(title) <- json \ "subtitle" if title.nonEmpty } yield structuredProperty(
title, title,
ModelConstants.SUBTITLE_QUALIFIER, null) ModelConstants.SUBTITLE_QUALIFIER, null)
result.setTitle((mainTitles ::: originalTitles ::: shortTitles ::: subtitles).asJava) result.setTitle((mainTitles ::: originalTitles ::: shortTitles ::: subtitles).asJava)
// DESCRIPTION // DESCRIPTION

3
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala
View File

@ -52,7 +52,8 @@ case class MAGPaper(
// List of authors // List of authors
authors: Option[List[MAGAuthor]], authors: Option[List[MAGAuthor]],
// List of Fields of Study // List of Fields of Study
fos: Option[List[MAGFieldOfStudy]] fos: Option[List[MAGFieldOfStudy]],
urls: Option[List[String]]
) )
case class MAGAuthor( case class MAGAuthor(

19
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkCreateMagDenormalizedTable.scala
View File

@ -146,7 +146,7 @@ class SparkCreateMagDenormalizedTable(propertyPath: String, args: Array[String],
$"Publisher".as("journalPublisher"), $"Publisher".as("journalPublisher"),
$"Webpage".as("journalWebpage") $"Webpage".as("journalWebpage")
) )
step3 val step4 = step3
.join(journals, step3("JournalId") === journals("JournalId"), "left") .join(journals, step3("JournalId") === journals("JournalId"), "left")
.select( .select(
step3("*"), step3("*"),
@ -155,6 +155,20 @@ class SparkCreateMagDenormalizedTable(propertyPath: String, args: Array[String],
journals("journalPublisher"), journals("journalPublisher"),
journals("journalWebpage") journals("journalWebpage")
) )
.cache
step4.count()
val paper_urls = MagUtility
.loadMagEntity(spark, "PaperUrls", magBasePath)
.groupBy("PaperId")
.agg(slice(collect_set("SourceUrl"), 1, 6).alias("urls"))
.cache
paper_urls.count
step4
.join(paper_urls, step4("PaperId") === paper_urls("PaperId"))
.select(step4("*"), paper_urls("urls"))
.select( .select(
$"PaperId".as("paperId"), $"PaperId".as("paperId"),
$"Rank".as("rank"), $"Rank".as("rank"),
@ -192,7 +206,8 @@ class SparkCreateMagDenormalizedTable(propertyPath: String, args: Array[String],
$"journalName".as("journalName"), $"journalName".as("journalName"),
$"journalIssn".as("journalIssn"), $"journalIssn".as("journalIssn"),
$"journalPublisher".as("journalPublisher"), $"journalPublisher".as("journalPublisher"),
$"journalWebpage".as("journalWebpage") $"journalWebpage".as("journalWebpage"),
$"urls"
) )
.write .write
.mode("OverWrite") .mode("OverWrite")