sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

completed mapping from paper to OAF, not tested

This commit is contained in:
Sandro La Bruzzo 2024-04-04 21:06:04 +02:00
parent 0794e0667b
commit 5142f462b5
3 changed files with 224 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

283
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala
View File

@ -2,7 +2,16 @@ package eu.dnetlib.dhp.collection.mag
import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.common.ModelConstants
import eu.dnetlib.dhp.schema.oaf import eu.dnetlib.dhp.schema.oaf
import eu.dnetlib.dhp.schema.oaf.{ Dataset => OafDataset, Author, DataInfo, Instance, Journal, Publication, Qualifier, Result} import eu.dnetlib.dhp.schema.oaf.{
Dataset => OafDataset,
Author,
DataInfo,
Instance,
Journal,
Publication,
Qualifier,
Result
}
import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils, PidType} import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils, PidType}
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils._ import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils._
import eu.dnetlib.dhp.utils import eu.dnetlib.dhp.utils
@ -26,7 +35,6 @@ case class MAGPaper(
date: Option[String], date: Option[String],
onlineDate: Option[String], onlineDate: Option[String],
publisher: Option[String], publisher: Option[String],
// Journal or Conference information (one will be populated)
journalId: Option[Long], journalId: Option[Long],
journalName: Option[String], journalName: Option[String],
journalIssn: Option[String], journalIssn: Option[String],
@ -50,7 +58,6 @@ case class MAGPaper(
docSubTypes: Option[String], docSubTypes: Option[String],
createdDate: Option[String], createdDate: Option[String],
abstractText: Option[String], abstractText: Option[String],
// List of authors
authors: Option[List[MAGAuthor]], authors: Option[List[MAGAuthor]],
urls: Option[List[String]] urls: Option[List[String]]
) )
@ -335,11 +342,8 @@ object MagUtility extends Serializable {
} }
def getInstanceType(magType: Option[String], source: Option[String]): Result = { def createResultFromType(magType: Option[String], source: Option[String]): Result = {
var result: Result = null
var result:Result = null
val di = new DataInfo val di = new DataInfo
di.setDeletedbyinference(false) di.setDeletedbyinference(false)
di.setInferred(false) di.setInferred(false)
@ -353,16 +357,18 @@ object MagUtility extends Serializable {
ModelConstants.DNET_PROVENANCE_ACTIONS ModelConstants.DNET_PROVENANCE_ACTIONS
) )
) )
if (magType== null) { if (magType == null) {
result = new Publication result = new Publication
result.setDataInfo(di) result.setDataInfo(di)
val i =new Instance val i = new Instance
i.setInstancetype(qualifier( i.setInstancetype(
"0038", qualifier(
"Other literature type", "0038",
ModelConstants.DNET_PUBLICATION_RESOURCE, "Other literature type",
ModelConstants.DNET_PUBLICATION_RESOURCE ModelConstants.DNET_PUBLICATION_RESOURCE,
)) ModelConstants.DNET_PUBLICATION_RESOURCE
)
)
result.setInstance(List(i).asJava) result.setInstance(List(i).asJava)
return result return result
@ -386,7 +392,7 @@ object MagUtility extends Serializable {
result = new Publication result = new Publication
qualifier("0043", "Journal", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE) qualifier("0043", "Journal", ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE)
case "patent" => case "patent" =>
if (source.nonEmpty) { if (source != null) {
val s = source.get.toLowerCase val s = source.get.toLowerCase
if (s.contains("patent") || s.contains("brevet")) { if (s.contains("patent") || s.contains("brevet")) {
result = new Publication result = new Publication
@ -405,8 +411,8 @@ object MagUtility extends Serializable {
ModelConstants.DNET_PUBLICATION_RESOURCE ModelConstants.DNET_PUBLICATION_RESOURCE
) )
} else if (s.contains("proceedings") || s.contains("conference") || s.contains("workshop") || s.contains( } else if (s.contains("proceedings") || s.contains("conference") || s.contains("workshop") || s.contains(
"symposium" "symposium"
)) { )) {
result = new Publication result = new Publication
qualifier( qualifier(
"0001", "0001",
@ -430,32 +436,32 @@ object MagUtility extends Serializable {
case "thesis" => case "thesis" =>
result = new Publication result = new Publication
qualifier( qualifier(
"0044", "0044",
"Thesis", "Thesis",
ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE,
ModelConstants.DNET_PUBLICATION_RESOURCE ModelConstants.DNET_PUBLICATION_RESOURCE
) )
case "dataset" => case "dataset" =>
result = new OafDataset result = new OafDataset
qualifier( qualifier(
"0021", "0021",
"Dataset", "Dataset",
ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE,
ModelConstants.DNET_PUBLICATION_RESOURCE ModelConstants.DNET_PUBLICATION_RESOURCE
) )
case "conference" => case "conference" =>
result = new Publication result = new Publication
qualifier( qualifier(
"0001", "0001",
"Article", "Article",
ModelConstants.DNET_PUBLICATION_RESOURCE, ModelConstants.DNET_PUBLICATION_RESOURCE,
ModelConstants.DNET_PUBLICATION_RESOURCE ModelConstants.DNET_PUBLICATION_RESOURCE
) )
} }
if (result != null) { if (result != null) {
result.setDataInfo(di) result.setDataInfo(di)
val i =new Instance val i = new Instance
i.setInstancetype(tp) i.setInstancetype(tp)
result.setInstance(List(i).asJava) result.setInstance(List(i).asJava)
} }
@ -463,60 +469,111 @@ object MagUtility extends Serializable {
} }
def convertMAGtoOAF(paper: MAGPaper): Publication = { def convertMAGtoOAF(paper: MAGPaper): Result = {
val pub = new Publication
val magPid = structuredProperty( // FILTER all the MAG paper with no URL
paper.doi.get, if (paper.urls == null || paper.urls.get != null || paper.urls.get.isEmpty)
qualifier( return null
PidType.mag_id.toString,
PidType.mag_id.toString, val result = createResultFromType(paper.docType, paper.originalVenue)
ModelConstants.DNET_PID_TYPES,
ModelConstants.DNET_PID_TYPES if (result == null)
), return null
null
val pidList = List(
structuredProperty(
paper.doi.get,
qualifier(
PidType.mag_id.toString,
PidType.mag_id.toString,
ModelConstants.DNET_PID_TYPES,
ModelConstants.DNET_PID_TYPES
),
null
)
) )
if (paper.doi.isDefined) { result.setPid(pidList.asJava)
pub.setPid(
result.setOriginalId(pidList.map(s => s.getValue).asJava)
result.setId(s"50|mag_________::${DHPUtils.md5(paper.paperId.get.toString)}")
val originalTitles = structuredProperty(paper.paperTitle.get, ModelConstants.MAIN_TITLE_QUALIFIER, null)
result.setTitle(List(originalTitles).asJava)
if (paper.date.orNull != null) {
result.setDateofacceptance(field(paper.date.get, null))
} else {
if (paper.year.isDefined && paper.year.get > 1700) {
result.setDateofacceptance(field(s"${paper.year.get}-01-01", null))
}
}
if (paper.onlineDate.orNull != null) {
result.setRelevantdate(
List( List(
magPid,
structuredProperty( structuredProperty(
paper.doi.get, paper.onlineDate.get,
qualifier( qualifier(
PidType.doi.toString, "published-online",
PidType.doi.toString, "published-online",
ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_DATACITE_DATE,
ModelConstants.DNET_PID_TYPES ModelConstants.DNET_DATACITE_DATE
), ),
null null
) )
).asJava ).asJava
) )
pub.setOriginalId(List(paper.paperId.get.toString, paper.doi.get).asJava)
} else {
pub.setPid(
List(
magPid
).asJava
)
pub.setOriginalId(List(paper.paperId.get.toString).asJava)
} }
pub.setId(s"50|mag_________::${DHPUtils.md5(paper.paperId.get.toString)}") if (paper.publisher.orNull != null) {
result.setPublisher(field(paper.publisher.get, null))
}
val mainTitles = structuredProperty(paper.originalTitle.get, ModelConstants.MAIN_TITLE_QUALIFIER, null) if (paper.date.isDefined)
result.setDateofacceptance(field(paper.date.get, null))
if (paper.onlineDate.orNull != null)
result.setRelevantdate(
List(
structuredProperty(
paper.onlineDate.get,
qualifier(
"published-online",
"published-online",
ModelConstants.DNET_DATACITE_DATE,
ModelConstants.DNET_DATACITE_DATE
),
null
)
).asJava
)
val originalTitles = structuredProperty(paper.paperTitle.get, ModelConstants.ALTERNATIVE_TITLE_QUALIFIER, null) if (paper.publisher.isDefined)
result.setPublisher(field(paper.publisher.get, null))
pub.setTitle(List(mainTitles, originalTitles).asJava) if (paper.journalId.isDefined && paper.journalName.isDefined) {
val j = new Journal
j.setName(paper.journalName.get)
j.setSp(paper.firstPage.orNull)
j.setEp(paper.lastPage.orNull)
if (paper.publisher.isDefined)
result.setPublisher(field(paper.publisher.get, null))
j.setIssnPrinted(paper.journalIssn.orNull)
j.setVol(paper.volume.orNull)
j.setIss(paper.issue.orNull)
j.setConferenceplace(paper.conferenceLocation.orNull)
result match {
case publication: Publication => publication.setJournal(j)
}
}
if (paper.bookTitle.isDefined)
pub.setSource(List(field[String](paper.bookTitle.get, null)).asJava)
if (paper.abstractText.isDefined) if (paper.abstractText.isDefined)
pub.setDescription(List(field(paper.abstractText.get, null)).asJava) result.setDescription(List(field(paper.abstractText.get, null)).asJava)
if (paper.authors.isDefined && paper.authors.get.nonEmpty) { if (paper.authors.isDefined && paper.authors.get.nonEmpty) {
pub.setAuthor( result.setAuthor(
paper.authors.get paper.authors.get
.filter(a => a.AuthorName.isDefined) .filter(a => a.AuthorName.isDefined)
.map(a => { .map(a => {
@ -528,30 +585,70 @@ object MagUtility extends Serializable {
) )
} }
if (paper.date.isDefined) val instance = result.getInstance().get(0)
pub.setDateofacceptance(field(paper.date.get, null)) instance.setPid(pidList.asJava)
instance.setAlternateIdentifier(
if (paper.publisher.isDefined) List(
pub.setPublisher(field(paper.publisher.get, null)) structuredProperty(
paper.doi.get,
if (paper.journalId.isDefined && paper.journalName.isDefined) { qualifier(
val j = new Journal PidType.doi.toString,
PidType.doi.toString,
j.setName(paper.journalName.get) ModelConstants.DNET_PID_TYPES,
j.setSp(paper.firstPage.orNull) ModelConstants.DNET_PID_TYPES
j.setEp(paper.lastPage.orNull) ),
if (paper.publisher.isDefined) null
pub.setPublisher(field(paper.publisher.get, null)) )
j.setIssnPrinted(paper.journalIssn.orNull) ).asJava
j.setVol(paper.volume.orNull) )
j.setIss(paper.issue.orNull) instance.setUrl(paper.urls.get.asJava)
j.setConferenceplace(paper.conferenceLocation.orNull) instance.setHostedby(ModelConstants.UNKNOWN_REPOSITORY)
j.setEdition(paper.conferenceName.orNull) instance.setAccessright(accessRight(
pub.setJournal(j) ModelConstants.UNKNOWN,
} ModelConstants.NOT_AVAILABLE,
ModelConstants.DNET_ACCESS_MODES,
pub ModelConstants.DNET_ACCESS_MODES
))
if (paper.authors.orNull != null && paper.authors.get.nonEmpty)
result.setAuthor(
paper.authors.get
.filter(a => a.AuthorName.orNull != null)
.map { a =>
val author = new Author
author.setFullname(a.AuthorName.get)
var authorPid = List(
structuredProperty(
a.AuthorId.get.toString,
qualifier(
PidType.mag_id.toString,
PidType.mag_id.toString,
ModelConstants.DNET_PID_TYPES,
ModelConstants.DNET_PID_TYPES
),
null
)
)
if (a.GridId.orNull != null) {
authorPid = authorPid ::: List(
structuredProperty(
a.AuthorId.get.toString,
qualifier(
PidType.mag_id.toString,
PidType.mag_id.toString,
ModelConstants.DNET_PID_TYPES,
ModelConstants.DNET_PID_TYPES
),
null
)
)
}
author.setPid(authorPid.asJava)
author
}
.asJava
)
result
} }
def convertInvertedIndexString(json_input: String): String = { def convertInvertedIndexString(json_input: String): String = {

20
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMAGtoOAF.scala
View File

@ -1,7 +1,8 @@
package eu.dnetlib.dhp.collection.mag package eu.dnetlib.dhp.collection.mag
import eu.dnetlib.dhp.application.AbstractScalaApplication import eu.dnetlib.dhp.application.AbstractScalaApplication
import org.apache.spark.sql.SparkSession import eu.dnetlib.dhp.schema.oaf.Result
import org.apache.spark.sql.{Encoders, SaveMode, SparkSession}
import org.slf4j.{Logger, LoggerFactory} import org.slf4j.{Logger, LoggerFactory}
class SparkMAGtoOAF(propertyPath: String, args: Array[String], log: Logger) class SparkMAGtoOAF(propertyPath: String, args: Array[String], log: Logger)
@ -20,9 +21,20 @@ class SparkMAGtoOAF(propertyPath: String, args: Array[String], log: Logger)
def convertMAG(spark: SparkSession, workingPath: String, mdStorePath: String): Unit = { def convertMAG(spark: SparkSession, workingPath: String, mdStorePath: String): Unit = {
import spark.implicits._ import spark.implicits._
val papers = spark.read.load(s"$workingPath/mag").as[MAGPaper] implicit val resultEncoder = Encoders.bean(classOf[Result])
val total = papers.count()
log.info(s"TOTAL PAPERS: $total") spark.read
.load(s"$workingPath/mag")
.as[MAGPaper].show()
spark.read
.load(s"$workingPath/mag")
.as[MAGPaper]
.map(s => MagUtility.convertMAGtoOAF(s))
.write
.mode(SaveMode.Overwrite)
.json(mdStorePath)
} }
} }

59
dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala
View File

@ -14,6 +14,7 @@ class MAGMappingTest {
val mapper = new ObjectMapper() val mapper = new ObjectMapper()
@Test
def mappingTest(): Unit = { def mappingTest(): Unit = {
val spark = SparkSession val spark = SparkSession
@ -22,13 +23,7 @@ class MAGMappingTest {
.master("local[*]") .master("local[*]")
.getOrCreate() .getOrCreate()
import spark.implicits._ new SparkMAGtoOAF(null,null,null).convertMAG(spark,"/Users/sandro/Downloads", "/Users/sandro/Downloads/mag_oaf")
val magDS = spark.read.load("/home/sandro/Downloads/mag").as[MAGPaper].where(col("journalId").isNotNull)
val paper = magDS.first()
print(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(MagUtility.convertMAGtoOAF(paper)))
} }
@ -36,47 +31,29 @@ class MAGMappingTest {
@Test @Test
def mappingMagType(): Unit = { def mappingMagType(): Unit = {
/*
+-----------+--------+
| docType| count|
+-----------+--------+
| null|79939635|
|BookChapter| 2431452|
| Dataset| 123923|
| Repository| 5044165|
| Thesis| 5525681|
| Conference| 5196866|
| Journal|89452763|
| Book| 4460017|
| Patent|64631955|
+-----------+--------+
"instancetype":{
"classid":"0001",
"classname":"Article",
"schemeid":"dnet:publication_resource",
"schemename":"dnet:publication_resource"},"instanceTypeMapping":[{"originalType":"journal-article","typeCode":null,"typeLabel":null,"vocabularyName":"openaire::coar_resource_types_3_1"}
*/
checkResult[Publication](MagUtility.getInstanceType(null, null), invisible = false,"Other literature type")
checkResult[Publication](MagUtility.getInstanceType(Some("BookChapter"), null), invisible = false,"Part of book or chapter of book")
checkResult[Publication](MagUtility.getInstanceType(Some("Book"), null), invisible = false,"Book")
checkResult[Publication](MagUtility.getInstanceType(Some("Repository"), null), invisible = true,"Other literature type")
checkResult[Publication](MagUtility.getInstanceType(Some("Thesis"), null), invisible = false,"Thesis")
checkResult[Publication](MagUtility.getInstanceType(Some("Conference"), null), invisible = false,"Article")
checkResult[Publication](MagUtility.getInstanceType(Some("Journal"), null), invisible = false,"Journal")
checkResult[Dataset](MagUtility.getInstanceType(Some("Dataset"), null), invisible = false,"Dataset")
checkResult[Publication](MagUtility.getInstanceType(Some("Patent"), Some("Patent Department of the Navy")), invisible = false,"Patent")
checkResult[Dataset](MagUtility.getInstanceType(Some("Dataset"), null), invisible = false,"Dataset")
checkResult[Publication](MagUtility.createResultFromType(null, null), invisible = false,"Other literature type")
checkResult[Publication](MagUtility.createResultFromType(Some("BookChapter"), null), invisible = false,"Part of book or chapter of book")
checkResult[Publication](MagUtility.createResultFromType(Some("Book"), null), invisible = false,"Book")
checkResult[Publication](MagUtility.createResultFromType(Some("Repository"), null), invisible = true,"Other literature type")
checkResult[Publication](MagUtility.createResultFromType(Some("Thesis"), null), invisible = false,"Thesis")
checkResult[Publication](MagUtility.createResultFromType(Some("Conference"), null), invisible = false,"Article")
checkResult[Publication](MagUtility.createResultFromType(Some("Journal"), null), invisible = false,"Journal")
checkResult[Dataset](MagUtility.createResultFromType(Some("Dataset"), null), invisible = false,"Dataset")
checkResult[Publication](MagUtility.createResultFromType(Some("Patent"), Some("Patent Department of the Navy")), invisible = false,"Patent")
checkResult[Publication](MagUtility.createResultFromType(Some("Patent"), Some("Brevet Department of the Navy")), invisible = false,"Patent")
checkResult[Publication](MagUtility.createResultFromType(Some("Patent"), Some("Journal of the Navy")), invisible = false,"Journal")
checkResult[Publication](MagUtility.createResultFromType(Some("Patent"), Some("Proceedings of the Navy")), invisible = false,"Article")
checkResult[Dataset](MagUtility.createResultFromType(Some("Dataset"), null), invisible = false,"Dataset")
assertNull(MagUtility.createResultFromType(Some("Patent"), null))
assertNull(MagUtility.createResultFromType(Some("Patent"), Some("Some name ")))
} }
def checkResult[T](r:Result, invisible:Boolean, typeName:String): Unit = { def checkResult[T](r:Result, invisible:Boolean, typeName:String): Unit = {
assertNotNull(r) assertNotNull(r)
assertTrue(r.isInstanceOf[T]) assertTrue(r.isInstanceOf[T])
assertNotNull(r.getDataInfo) assertNotNull(r.getDataInfo)