package eu.dnetlib.dhp.bioschema import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup import eu.dnetlib.dhp.bioschema.BioschemaModelConstants._ import eu.dnetlib.dhp.schema.action.AtomicAction import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.utils.{IdentifierFactory, OafMapperUtils} import eu.dnetlib.dhp.schema.oaf.{Dataset => OafDataset, _} import eu.dnetlib.dhp.utils.DHPUtils import org.apache.commons.lang3.StringUtils import org.json4s.DefaultFormats import org.json4s.JsonAST.{JField, JObject, JString} import org.json4s.jackson.JsonMethods.parse import java.text.SimpleDateFormat import java.time.LocalDate import java.time.chrono.ThaiBuddhistDate import java.time.format.DateTimeFormatter import java.util.{Date, Locale} import scala.collection.JavaConverters._ import scala.io.{Codec, Source} object BioschemaToOAFTransformation { val mapper = new ObjectMapper() val DATA_INFO: DataInfo = OafMapperUtils.dataInfo( false, null, false, false, ModelConstants.PROVENANCE_ACTION_SET_QUALIFIER, "0.9" ) val collectedFromMap: Map[String, KeyValue] = { val PEDCollectedFrom: KeyValue = OafMapperUtils.keyValue( "10|ped_________::changeme", "PED" ) PEDCollectedFrom.setDataInfo(DATA_INFO) Map( "ped" -> PEDCollectedFrom ) } /** This method should skip record if json contains invalid text * defined in gile datacite_filter * * @param json * @return True if the record should be skipped */ def skip_record(json: String): Boolean = { datacite_filter.exists(f => json.contains(f)) } @deprecated("this method will be removed", "dhp") def toActionSet(item: Oaf): (String, String) = { val mapper = new ObjectMapper() item match { case dataset: OafDataset => val a: AtomicAction[OafDataset] = new AtomicAction[OafDataset] a.setClazz(classOf[OafDataset]) a.setPayload(dataset) (dataset.getClass.getCanonicalName, mapper.writeValueAsString(a)) case publication: Publication => val a: AtomicAction[Publication] = new AtomicAction[Publication] a.setClazz(classOf[Publication]) a.setPayload(publication) (publication.getClass.getCanonicalName, mapper.writeValueAsString(a)) case software: Software => val a: AtomicAction[Software] = new AtomicAction[Software] a.setClazz(classOf[Software]) a.setPayload(software) (software.getClass.getCanonicalName, mapper.writeValueAsString(a)) case orp: OtherResearchProduct => val a: AtomicAction[OtherResearchProduct] = new AtomicAction[OtherResearchProduct] a.setClazz(classOf[OtherResearchProduct]) a.setPayload(orp) (orp.getClass.getCanonicalName, mapper.writeValueAsString(a)) case relation: Relation => val a: AtomicAction[Relation] = new AtomicAction[Relation] a.setClazz(classOf[Relation]) a.setPayload(relation) (relation.getClass.getCanonicalName, mapper.writeValueAsString(a)) case _ => null } } def embargo_end(embargo_end_date: String): Boolean = { val dt = LocalDate.parse(embargo_end_date, DateTimeFormatter.ofPattern("[yyyy-MM-dd]")) val td = LocalDate.now() td.isAfter(dt) } def extract_date(input: String): Option[String] = { val d = Date_regex .map(pattern => { val matcher = pattern.matcher(input) if (matcher.find()) matcher.group(0) else null }) .find(s => s != null) if (d.isDefined) { val a_date = if (d.get.length == 4) s"01-01-${d.get}" else d.get try { return Some(LocalDate.parse(a_date, df_en).toString) } catch { case _: Throwable => try { return Some(LocalDate.parse(a_date, df_it).toString) } catch { case _: Throwable => return None } } } d } def fix_thai_date(input: String, format: String): String = { try { val a_date = LocalDate.parse(input, DateTimeFormatter.ofPattern(format)) val d = ThaiBuddhistDate.of(a_date.getYear, a_date.getMonth.getValue, a_date.getDayOfMonth) LocalDate.from(d).toString } catch { case _: Throwable => "" } } def getTypeQualifier( resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup ): (Qualifier, Qualifier) = { if (resourceType != null && resourceType.nonEmpty) { val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceType) if (typeQualifier != null) return ( typeQualifier, vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid ) ) } if (schemaOrg != null && schemaOrg.nonEmpty) { val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, schemaOrg) if (typeQualifier != null) return ( typeQualifier, vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid ) ) } if (resourceTypeGeneral != null && resourceTypeGeneral.nonEmpty) { val typeQualifier = vocabularies.getSynonymAsQualifier( ModelConstants.DNET_PUBLICATION_RESOURCE, resourceTypeGeneral ) if (typeQualifier != null) return ( typeQualifier, vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid ) ) } null } def getResult( resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup ): Result = { val typeQualifiers: (Qualifier, Qualifier) = getTypeQualifier(resourceType, resourceTypeGeneral, schemaOrg, vocabularies) if (typeQualifiers == null) return null val i = new Instance i.setInstancetype(typeQualifiers._1) typeQualifiers._2.getClassname match { case "dataset" => val r = new OafDataset r.setInstance(List(i).asJava) return r case "publication" => val r = new Publication r.setInstance(List(i).asJava) return r case "software" => val r = new Software r.setInstance(List(i).asJava) return r case "other" => val r = new OtherResearchProduct r.setInstance(List(i).asJava) return r } null } def available_date(input: String): Boolean = { implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(input) val l: List[String] = for { JObject(dates) <- json \\ "dates" JField("dateType", JString(dateTypes)) <- dates } yield dateTypes l.exists(p => p.equalsIgnoreCase("available")) } def createDNetTargetIdentifier(pid: String, pidType: String, idPrefix: String): String = { val f_part = s"$idPrefix|${pidType.toLowerCase}".padTo(15, '_') s"$f_part::${IdentifierFactory.md5(pid.toLowerCase)}" } def generateOAFDate(dt: String, q: Qualifier): StructuredProperty = { OafMapperUtils.structuredProperty(dt, q, null) } def generateRelation( sourceId: String, targetId: String, relClass: String, cf: KeyValue, di: DataInfo ): Relation = { val r = new Relation r.setSource(sourceId) r.setTarget(targetId) r.setRelType(ModelConstants.RESULT_PROJECT) r.setRelClass(relClass) r.setSubRelType(ModelConstants.OUTCOME) r.setCollectedfrom(List(cf).asJava) r.setDataInfo(di) r } def get_projectRelation(awardUri: String, sourceId: String): List[Relation] = { val match_pattern = funder_regex.find(s => s._1.matcher(awardUri).find()) if (match_pattern.isDefined) { val m = match_pattern.get._1 val p = match_pattern.get._2 val grantId = m.matcher(awardUri).replaceAll("$2") val targetId = s"$p${DHPUtils.md5(grantId)}" List(generateRelation(sourceId, targetId, "isProducedBy", collectedFromMap("ped"), dataInfo)) } else List() } def generateOAF( input: String, ts: Long, dateOfCollection: Long, vocabularies: VocabularyGroup, exportLinks: Boolean ): List[Oaf] = { if (skip_record(input)) return List() implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json = parse(input) val resourceType = (json \ "types" \ "resourceType").extractOrElse[String](null) val resourceTypeGeneral = (json \ "types" \ "resourceTypeGeneral").extractOrElse[String](null) val schemaOrg = (json \ "types" \ "schemaOrg").extractOrElse[String](null) //Mapping type based on vocabularies dnet:publication_resource and dnet:result_typologies val result = getResult(resourceType, resourceTypeGeneral, schemaOrg, vocabularies) if (result == null) return List() val pid = (json \ "id").extract[String] result.setPid( List( OafMapperUtils.structuredProperty( pid, "ped", "ped", ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES, DATA_INFO ) ).asJava ) result.setId(OafMapperUtils.createOpenaireId(50, s"ped_________::$pid", true)) result.setOriginalId(List(pid).asJava) result.setDataInfo(dataInfo) val creators = (json \\ "creators").extractOrElse[List[CreatorType]](List()) val titles: List[TitleType] = (json \\ "titles").extractOrElse[List[TitleType]](List()) result.setTitle( titles .filter(t => t.title.nonEmpty) .map(t => { if (t.titleType.isEmpty) { OafMapperUtils .structuredProperty(t.title.get, ModelConstants.MAIN_TITLE_QUALIFIER, null) } else { OafMapperUtils.structuredProperty( t.title.get, t.titleType.get, t.titleType.get, ModelConstants.DNET_DATACITE_TITLE, ModelConstants.DNET_DATACITE_TITLE, null ) } }) .asJava ) val dates = (json \\ "dates").extract[List[DateType]] val publication_year = (json \\ "publicationYear").extractOrElse[String](null) val i_date = dates .filter(d => d.date.isDefined && d.dateType.isDefined) .find(d => d.dateType.get.equalsIgnoreCase("issued")) .map(d => extract_date(d.date.get)) val a_date: Option[String] = dates .filter(d => d.date.isDefined && d.dateType.isDefined && d.dateType.get.equalsIgnoreCase("available")) .map(d => extract_date(d.date.get)) .find(d => d != null && d.isDefined) .map(d => d.get) if (a_date.isDefined) { result.setEmbargoenddate(OafMapperUtils.field(a_date.get, null)) } if (i_date.isDefined && i_date.get.isDefined) { result.setDateofacceptance(OafMapperUtils.field(i_date.get.get, null)) result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(i_date.get.get, null)) } else if (publication_year != null) { result.setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null)) result .getInstance() .get(0) .setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null)) } result.setRelevantdate( dates .filter(d => d.date.isDefined && d.dateType.isDefined) .map(d => (extract_date(d.date.get), d.dateType.get)) .filter(d => d._1.isDefined) .map(d => ( d._1.get, vocabularies.getTermAsQualifier(ModelConstants.DNET_DATACITE_DATE, d._2.toLowerCase()) ) ) .filter(d => d._2 != null) .map(d => generateOAFDate(d._1, d._2)) .asJava ) result.setCollectedfrom(List(collectedFromMap("ped")).asJava) val descriptions = (json \\ "descriptions").extract[List[DescriptionType]] result.setDescription( descriptions .filter(d => d.description.isDefined) .map(d => OafMapperUtils.field(d.description.get, null)) .filter(s => s != null) .asJava ) val publisher = (json \\ "publisher").extractOrElse[String](null) if (publisher != null) result.setPublisher(OafMapperUtils.field(publisher, null)) val language: String = (json \\ "language").extractOrElse[String](null) if (language != null) result.setLanguage( vocabularies.getSynonymAsQualifier(ModelConstants.DNET_LANGUAGES, language) ) val instance = result.getInstance().get(0) val accessRights: List[String] = for { JObject(rightsList) <- json \\ "rightsList" JField("rightsUri", JString(rightsUri)) <- rightsList } yield rightsUri val aRights: Option[AccessRight] = accessRights .map(r => { vocabularies.getSynonymAsQualifier(ModelConstants.DNET_ACCESS_MODES, r) }) .find(q => q != null) .map(q => { val a = new AccessRight a.setClassid(q.getClassid) a.setClassname(q.getClassname) a.setSchemeid(q.getSchemeid) a.setSchemename(q.getSchemename) a }) val access_rights_qualifier = if (aRights.isDefined) aRights.get else OafMapperUtils.accessRight( ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE, ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES ) instance.setCollectedfrom(collectedFromMap("ped")) instance.setUrl(List(s"https://proteinensemble.org/$pid").asJava) instance.setAccessright(access_rights_qualifier) instance.setPid(result.getPid) val license = accessRights .find(r => r.startsWith("http") && r.matches( ".*(/licenses|/publicdomain|unlicense\\.org/|/legal-and-data-protection-notices|/download/license|/open-government-licence).*" ) ) if (license.isDefined) instance.setLicense(OafMapperUtils.field(license.get, null)) val awardUris: List[String] = for { JObject(fundingReferences) <- json \\ "fundingReferences" JField("awardUri", JString(awardUri)) <- fundingReferences } yield awardUri result.setId(IdentifierFactory.createIdentifier(result)) var relations: List[Relation] = awardUris.flatMap(a => get_projectRelation(a, result.getId)).filter(r => r != null) if (result.getId == null) return List() if (exportLinks) { val rels: List[RelatedIdentifierType] = for { JObject(relIdentifier) <- json \\ "relatedIdentifiers" JField("relationType", JString(relationType)) <- relIdentifier JField("relatedIdentifierType", JString(relatedIdentifierType)) <- relIdentifier JField("relatedIdentifier", JString(relatedIdentifier)) <- relIdentifier } yield RelatedIdentifierType(relationType, relatedIdentifier, relatedIdentifierType) relations = relations ::: generateRelations( rels, result.getId, if (i_date.isDefined && i_date.get.isDefined) i_date.get.get else null ) } if (relations != null && relations.nonEmpty) { List(result) ::: relations } else List(result) } private def generateRelations( rels: List[RelatedIdentifierType], id: String, date: String ): List[Relation] = { rels .filter(r => subRelTypeMapping .contains(r.relationType) && (r.relatedIdentifierType.equalsIgnoreCase("doi") || r.relatedIdentifierType.equalsIgnoreCase("pmid") || r.relatedIdentifierType.equalsIgnoreCase("arxiv")) ) .map(r => { val rel = new Relation rel.setCollectedfrom(List(collectedFromMap("ped")).asJava) rel.setDataInfo(dataInfo) val subRelType = subRelTypeMapping(r.relationType).relType rel.setRelType(REL_TYPE_VALUE) rel.setSubRelType(subRelType) rel.setRelClass(r.relationType) val dateProps: KeyValue = OafMapperUtils.keyValue(DATE_RELATION_KEY, date) rel.setProperties(List(dateProps).asJava) rel.setSource(id) rel.setTarget( DHPUtils.generateUnresolvedIdentifier(r.relatedIdentifier, r.relatedIdentifierType) ) rel.setCollectedfrom(List(collectedFromMap("ped")).asJava) rel.getCollectedfrom.asScala.map(c => c.getValue).toList rel }) } def generateDSId(input: String): String = { val b = StringUtils.substringBefore(input, "::") val a = StringUtils.substringAfter(input, "::") s"10|$b::${DHPUtils.md5(a)}" } }