D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +33 -1019
2 changed files with 33 additions and 1019 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b37bc277c4 - Show all commits

1021
dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json
View File

File diff suppressed because it is too large Load Diff

31
dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala
View File

@ -19,11 +19,33 @@ import java.time.chrono.ThaiBuddhistDate
import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatter
import java.util.{Date, Locale} import java.util.{Date, Locale}
import scala.collection.JavaConverters._ import scala.collection.JavaConverters._
import scala.io.{Codec, Source}
object DataciteToOAFTransformation { object DataciteToOAFTransformation {
case class HostedByMapType(
openaire_id: String,
datacite_name: String,
official_name: String,
similarity: Option[Float]
) {}
val mapper = new ObjectMapper() val mapper = new ObjectMapper()
val unknown_repository: HostedByMapType = HostedByMapType(
ModelConstants.UNKNOWN_REPOSITORY_ORIGINALID,
ModelConstants.UNKNOWN_REPOSITORY.getValue,
ModelConstants.UNKNOWN_REPOSITORY.getValue,
Some(1.0f)
)
val hostedByMap: Map[String, HostedByMapType] = {
val s = Source.fromInputStream(getClass.getResourceAsStream("hostedBy_map.json")).mkString
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
lazy val json: org.json4s.JValue = parse(s)
json.extract[Map[String, HostedByMapType]]
}
/** This method should skip record if json contains invalid text /** This method should skip record if json contains invalid text
* defined in gile datacite_filter * defined in gile datacite_filter
* *
@ -534,12 +556,9 @@ object DataciteToOAFTransformation {
if (client.isDefined) { if (client.isDefined) {
instance.setHostedby( val hb = hostedByMap.getOrElse(client.get.toUpperCase(), unknown_repository)
OafMapperUtils.keyValue( instance.setHostedby(OafMapperUtils.keyValue(generateDSId(hb.openaire_id), hb.official_name))
generateDSId(ModelConstants.UNKNOWN_REPOSITORY_ORIGINALID),
ModelConstants.UNKNOWN_REPOSITORY.getValue
)
)
instance.setCollectedfrom(DATACITE_COLLECTED_FROM) instance.setCollectedfrom(DATACITE_COLLECTED_FROM)
instance.setUrl(List(s"https://dx.doi.org/$doi").asJava) instance.setUrl(List(s"https://dx.doi.org/$doi").asJava)
instance.setAccessright(access_rights_qualifier) instance.setAccessright(access_rights_qualifier)