usage-stats-export-wf-v2 #117
|
|
@ -23,5 +23,4 @@
|
|||
/build
|
||||
spark-warehouse
|
||||
/**/job-override.properties
|
||||
/**/*.log
|
||||
|
||||
/**/*.log
|
||||
|
|
@ -115,6 +115,8 @@ public class AuthorMerger {
|
|||
}
|
||||
|
||||
public static String pidToComparableString(StructuredProperty pid) {
|
||||
if (pid == null)
|
||||
return "";
|
||||
return (pid.getQualifier() != null
|
||||
? pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""
|
||||
: "")
|
||||
|
|
|
|||
|
|
@ -7,6 +7,37 @@
|
|||
<version>1.2.4-SNAPSHOT</version>
|
||||
</parent>
|
||||
<artifactId>dhp-aggregation</artifactId>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>net.alchim31.maven</groupId>
|
||||
<artifactId>scala-maven-plugin</artifactId>
|
||||
<version>4.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>scala-compile-first</id>
|
||||
<phase>initialize</phase>
|
||||
<goals>
|
||||
<goal>add-source</goal>
|
||||
<goal>compile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>scala-test-compile</id>
|
||||
<phase>process-test-resources</phase>
|
||||
<goals>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<scalaVersion>${scala.version}</scalaVersion>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
||||
|
||||
<dependencies>
|
||||
|
|
@ -24,12 +55,6 @@
|
|||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.sun.xml.bind</groupId>
|
||||
<artifactId>jaxb-core</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -37,6 +62,13 @@
|
|||
<artifactId>dhp-schemas</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-graph-mapper</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sf.saxon</groupId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,544 @@
|
|||
package eu.dnetlib.dhp.actionmanager.datacite
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup
|
||||
import eu.dnetlib.dhp.schema.action.AtomicAction
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants
|
||||
import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, KeyValue, Oaf, OafMapperUtils, OtherResearchProduct, Publication, Qualifier, Relation, Result, Software, StructuredProperty, Dataset => OafDataset}
|
||||
import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, KeyValue, Oaf, OtherResearchProduct, Publication, Qualifier, Relation, Result, Software, StructuredProperty, Dataset => OafDataset}
|
||||
import eu.dnetlib.dhp.utils.DHPUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.json4s.DefaultFormats
|
||||
import org.json4s.JsonAST.{JField, JObject, JString}
|
||||
import org.json4s.jackson.JsonMethods.parse
|
||||
|
||||
import java.nio.charset.CodingErrorAction
|
||||
import java.text.SimpleDateFormat
|
||||
import java.time.LocalDate
|
||||
import java.time.chrono.ThaiBuddhistDate
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.util.{Date, Locale}
|
||||
import java.util.regex.Pattern
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.io.{Codec, Source}
|
||||
|
||||
case class DataciteType(doi: String, timestamp: Long, isActive: Boolean, json: String) {}
|
||||
|
||||
case class NameIdentifiersType(nameIdentifierScheme: Option[String], schemeUri: Option[String], nameIdentifier: Option[String]) {}
|
||||
|
||||
case class CreatorType(nameType: Option[String], nameIdentifiers: Option[List[NameIdentifiersType]], name: Option[String], familyName: Option[String], givenName: Option[String], affiliation: Option[List[String]]) {}
|
||||
|
||||
case class TitleType(title: Option[String], titleType: Option[String], lang: Option[String]) {}
|
||||
|
||||
case class SubjectType(subject: Option[String], subjectScheme: Option[String]) {}
|
||||
|
||||
case class DescriptionType(descriptionType: Option[String], description: Option[String]) {}
|
||||
|
||||
case class FundingReferenceType(funderIdentifierType: Option[String], awardTitle: Option[String], awardUri: Option[String], funderName: Option[String], funderIdentifier: Option[String], awardNumber: Option[String]) {}
|
||||
|
||||
case class DateType(date: Option[String], dateType: Option[String]) {}
|
||||
|
||||
case class HostedByMapType(openaire_id: String, datacite_name: String, official_name: String, similarity: Option[Float]) {}
|
||||
|
||||
object DataciteToOAFTransformation {
|
||||
val UNKNOWN_REPOSITORY_ORIGINALID = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18"
|
||||
val DATACITE_ID = "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"
|
||||
|
||||
val DNET_DATACITE_DATE = "dnet:dataCite_date"
|
||||
|
||||
val DNET_DATACITE_TITLE = "dnet:dataCite_title"
|
||||
val SYSIMPORT_ACTIONSET = "sysimport:actionset"
|
||||
val DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"
|
||||
|
||||
val PROVENANCE_ACTION_SET_QUALIFIER: Qualifier = OafMapperUtils.qualifier(SYSIMPORT_ACTIONSET, SYSIMPORT_ACTIONSET, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS)
|
||||
|
||||
val MAIN_TITLE_QUALIFIER:Qualifier = OafMapperUtils.qualifier("main title","main title",DNET_DATACITE_TITLE,DNET_DATACITE_TITLE)
|
||||
|
||||
implicit val codec: Codec = Codec("UTF-8")
|
||||
codec.onMalformedInput(CodingErrorAction.REPLACE)
|
||||
codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
|
||||
|
||||
val DOI_CLASS = "doi"
|
||||
val SUBJ_CLASS = "keywords"
|
||||
|
||||
|
||||
val j_filter: List[String] = {
|
||||
val s = Source.fromInputStream(getClass.getResourceAsStream("datacite_filter")).mkString
|
||||
s.lines.toList
|
||||
}
|
||||
|
||||
val mapper = new ObjectMapper()
|
||||
val unknown_repository: HostedByMapType = HostedByMapType(UNKNOWN_REPOSITORY_ORIGINALID, ModelConstants.UNKNOWN_REPOSITORY.getValue, ModelConstants.UNKNOWN_REPOSITORY.getValue, Some(1.0F))
|
||||
|
||||
val dataInfo: DataInfo = generateDataInfo("0.9")
|
||||
val DATACITE_COLLECTED_FROM: KeyValue = OafMapperUtils.keyValue(DATACITE_ID, "Datacite")
|
||||
|
||||
val hostedByMap: Map[String, HostedByMapType] = {
|
||||
val s = Source.fromInputStream(getClass.getResourceAsStream("hostedBy_map.json")).mkString
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
lazy val json: org.json4s.JValue = parse(s)
|
||||
json.extract[Map[String, HostedByMapType]]
|
||||
}
|
||||
|
||||
val df_en: DateTimeFormatter = DateTimeFormatter.ofPattern("[MM-dd-yyyy][MM/dd/yyyy][dd-MM-yy][dd-MMM-yyyy][dd/MMM/yyyy][dd-MMM-yy][dd/MMM/yy][dd-MM-yy][dd/MM/yy][dd-MM-yyyy][dd/MM/yyyy][yyyy-MM-dd][yyyy/MM/dd]", Locale.ENGLISH)
|
||||
val df_it: DateTimeFormatter = DateTimeFormatter.ofPattern("[dd-MM-yyyy][dd/MM/yyyy]", Locale.ITALIAN)
|
||||
|
||||
val funder_regex: List[(Pattern, String)] = List(
|
||||
(Pattern.compile("(info:eu-repo/grantagreement/ec/h2020/)(\\d\\d\\d\\d\\d\\d)(.*)", Pattern.MULTILINE | Pattern.CASE_INSENSITIVE), "40|corda__h2020::"),
|
||||
(Pattern.compile("(info:eu-repo/grantagreement/ec/fp7/)(\\d\\d\\d\\d\\d\\d)(.*)", Pattern.MULTILINE | Pattern.CASE_INSENSITIVE), "40|corda_______::")
|
||||
|
||||
)
|
||||
|
||||
val Date_regex: List[Pattern] = List(
|
||||
//Y-M-D
|
||||
Pattern.compile("(18|19|20)\\d\\d([- /.])(0[1-9]|1[012])\\2(0[1-9]|[12][0-9]|3[01])", Pattern.MULTILINE),
|
||||
//M-D-Y
|
||||
Pattern.compile("((0[1-9]|1[012])|([1-9]))([- /.])(0[1-9]|[12][0-9]|3[01])([- /.])(18|19|20)?\\d\\d", Pattern.MULTILINE),
|
||||
//D-M-Y
|
||||
Pattern.compile("(?:(?:31(/|-|\\.)(?:0?[13578]|1[02]|(?:Jan|Mar|May|Jul|Aug|Oct|Dec)))\\1|(?:(?:29|30)(/|-|\\.)(?:0?[1,3-9]|1[0-2]|(?:Jan|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))\\2))(?:(?:1[6-9]|[2-9]\\d)?\\d{2})|(?:29(/|-|\\.)(?:0?2|(?:Feb))\\3(?:(?:(?:1[6-9]|[2-9]\\d)?(?:0[48]|[2468][048]|[13579][26])|(?:(?:16|[2468][048]|[3579][26])00))))|(?:0?[1-9]|1\\d|2[0-8])(/|-|\\.)(?:(?:0?[1-9]|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep))|(?:1[0-2]|(?:Oct|Nov|Dec)))\\4(?:(?:1[6-9]|[2-9]\\d)?\\d{2})", Pattern.MULTILINE),
|
||||
//Y
|
||||
Pattern.compile("(19|20)\\d\\d", Pattern.MULTILINE)
|
||||
)
|
||||
|
||||
|
||||
def filter_json(json: String): Boolean = {
|
||||
j_filter.exists(f => json.contains(f))
|
||||
}
|
||||
|
||||
def toActionSet(item: Oaf): (String, String) = {
|
||||
val mapper = new ObjectMapper()
|
||||
|
||||
item match {
|
||||
case dataset: OafDataset =>
|
||||
val a: AtomicAction[OafDataset] = new AtomicAction[OafDataset]
|
||||
a.setClazz(classOf[OafDataset])
|
||||
a.setPayload(dataset)
|
||||
(dataset.getClass.getCanonicalName, mapper.writeValueAsString(a))
|
||||
case publication: Publication =>
|
||||
val a: AtomicAction[Publication] = new AtomicAction[Publication]
|
||||
a.setClazz(classOf[Publication])
|
||||
a.setPayload(publication)
|
||||
(publication.getClass.getCanonicalName, mapper.writeValueAsString(a))
|
||||
case software: Software =>
|
||||
val a: AtomicAction[Software] = new AtomicAction[Software]
|
||||
a.setClazz(classOf[Software])
|
||||
a.setPayload(software)
|
||||
(software.getClass.getCanonicalName, mapper.writeValueAsString(a))
|
||||
case orp: OtherResearchProduct =>
|
||||
val a: AtomicAction[OtherResearchProduct] = new AtomicAction[OtherResearchProduct]
|
||||
a.setClazz(classOf[OtherResearchProduct])
|
||||
a.setPayload(orp)
|
||||
(orp.getClass.getCanonicalName, mapper.writeValueAsString(a))
|
||||
|
||||
case relation: Relation =>
|
||||
val a: AtomicAction[Relation] = new AtomicAction[Relation]
|
||||
a.setClazz(classOf[Relation])
|
||||
a.setPayload(relation)
|
||||
(relation.getClass.getCanonicalName, mapper.writeValueAsString(a))
|
||||
case _ =>
|
||||
null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def embargo_end(embargo_end_date: String): Boolean = {
|
||||
val dt = LocalDate.parse(embargo_end_date, DateTimeFormatter.ofPattern("[yyyy-MM-dd]"))
|
||||
val td = LocalDate.now()
|
||||
td.isAfter(dt)
|
||||
}
|
||||
|
||||
|
||||
def extract_date(input: String): Option[String] = {
|
||||
val d = Date_regex.map(pattern => {
|
||||
val matcher = pattern.matcher(input)
|
||||
if (matcher.find())
|
||||
matcher.group(0)
|
||||
else
|
||||
null
|
||||
}
|
||||
).find(s => s != null)
|
||||
|
||||
if (d.isDefined) {
|
||||
val a_date = if (d.get.length == 4) s"01-01-${d.get}" else d.get
|
||||
try {
|
||||
return Some(LocalDate.parse(a_date, df_en).toString)
|
||||
} catch {
|
||||
case _: Throwable => try {
|
||||
return Some(LocalDate.parse(a_date, df_it).toString)
|
||||
} catch {
|
||||
case _: Throwable =>
|
||||
return None
|
||||
}
|
||||
}
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
|
||||
def fix_thai_date(input:String, format:String) :String = {
|
||||
try {
|
||||
val a_date = LocalDate.parse(input,DateTimeFormatter.ofPattern(format))
|
||||
val d = ThaiBuddhistDate.of(a_date.getYear, a_date.getMonth.getValue, a_date.getDayOfMonth)
|
||||
LocalDate.from(d).toString
|
||||
} catch {
|
||||
case _: Throwable => ""
|
||||
}
|
||||
}
|
||||
|
||||
def getTypeQualifier(resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup): (Qualifier, Qualifier) = {
|
||||
if (resourceType != null && resourceType.nonEmpty) {
|
||||
val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceType)
|
||||
if (typeQualifier != null)
|
||||
return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
|
||||
}
|
||||
if (schemaOrg != null && schemaOrg.nonEmpty) {
|
||||
val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, schemaOrg)
|
||||
if (typeQualifier != null)
|
||||
return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
|
||||
|
||||
}
|
||||
if (resourceTypeGeneral != null && resourceTypeGeneral.nonEmpty) {
|
||||
val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceTypeGeneral)
|
||||
if (typeQualifier != null)
|
||||
return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
|
||||
|
||||
}
|
||||
null
|
||||
}
|
||||
|
||||
|
||||
def getResult(resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup): Result = {
|
||||
val typeQualifiers: (Qualifier, Qualifier) = getTypeQualifier(resourceType, resourceTypeGeneral, schemaOrg, vocabularies)
|
||||
if (typeQualifiers == null)
|
||||
return null
|
||||
val i = new Instance
|
||||
i.setInstancetype(typeQualifiers._1)
|
||||
typeQualifiers._2.getClassname match {
|
||||
case "dataset" =>
|
||||
val r = new OafDataset
|
||||
r.setInstance(List(i).asJava)
|
||||
return r
|
||||
case "publication" =>
|
||||
val r = new Publication
|
||||
r.setInstance(List(i).asJava)
|
||||
return r
|
||||
case "software" =>
|
||||
val r = new Software
|
||||
r.setInstance(List(i).asJava)
|
||||
return r
|
||||
case "other" =>
|
||||
val r = new OtherResearchProduct
|
||||
r.setInstance(List(i).asJava)
|
||||
return r
|
||||
}
|
||||
null
|
||||
}
|
||||
|
||||
|
||||
def available_date(input: String): Boolean = {
|
||||
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
lazy val json: org.json4s.JValue = parse(input)
|
||||
val l: List[String] = for {
|
||||
JObject(dates) <- json \\ "dates"
|
||||
JField("dateType", JString(dateTypes)) <- dates
|
||||
} yield dateTypes
|
||||
|
||||
l.exists(p => p.equalsIgnoreCase("available"))
|
||||
|
||||
}
|
||||
|
||||
|
||||
def OPEN_ACCESS_RIGHT = {
|
||||
val result = new Qualifier
|
||||
result.setClassid("OPEN")
|
||||
result.setClassid("OPEN")
|
||||
result.setSchemeid(ModelConstants.DNET_ACCESS_MODES)
|
||||
result.setSchemename(ModelConstants.DNET_ACCESS_MODES)
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* As describe in ticket #6377
|
||||
* when the result come from figshare we need to remove subject
|
||||
* and set Access rights OPEN.
|
||||
* @param r
|
||||
*/
|
||||
def fix_figshare(r: Result): Unit = {
|
||||
|
||||
if (r.getInstance() != null) {
|
||||
val hosted_by_figshare = r.getInstance().asScala.exists(i => i.getHostedby != null && "figshare".equalsIgnoreCase(i.getHostedby.getValue))
|
||||
if (hosted_by_figshare) {
|
||||
r.getInstance().asScala.foreach(i => i.setAccessright(OPEN_ACCESS_RIGHT))
|
||||
val l: List[StructuredProperty] = List()
|
||||
r.setSubject(l.asJava)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def generateOAFDate(dt: String, q: Qualifier): StructuredProperty = {
|
||||
OafMapperUtils.structuredProperty(dt, q, null)
|
||||
}
|
||||
|
||||
def generateRelation(sourceId: String, targetId: String, relClass: String, cf: KeyValue, di: DataInfo): Relation = {
|
||||
|
||||
val r = new Relation
|
||||
r.setSource(sourceId)
|
||||
r.setTarget(targetId)
|
||||
r.setRelType(ModelConstants.RESULT_PROJECT)
|
||||
r.setRelClass(relClass)
|
||||
r.setSubRelType(ModelConstants.OUTCOME)
|
||||
r.setCollectedfrom(List(cf).asJava)
|
||||
r.setDataInfo(di)
|
||||
r
|
||||
|
||||
|
||||
}
|
||||
|
||||
def get_projectRelation(awardUri: String, sourceId: String): List[Relation] = {
|
||||
val match_pattern = funder_regex.find(s => s._1.matcher(awardUri).find())
|
||||
|
||||
if (match_pattern.isDefined) {
|
||||
val m = match_pattern.get._1
|
||||
val p = match_pattern.get._2
|
||||
val grantId = m.matcher(awardUri).replaceAll("$2")
|
||||
val targetId = s"$p${DHPUtils.md5(grantId)}"
|
||||
List(
|
||||
generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo),
|
||||
generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
|
||||
)
|
||||
}
|
||||
else
|
||||
List()
|
||||
|
||||
}
|
||||
|
||||
|
||||
def generateOAF(input: String, ts: Long, dateOfCollection: Long, vocabularies: VocabularyGroup): List[Oaf] = {
|
||||
if (filter_json(input))
|
||||
return List()
|
||||
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
lazy val json = parse(input)
|
||||
|
||||
val resourceType = (json \ "attributes" \ "types" \ "resourceType").extractOrElse[String](null)
|
||||
val resourceTypeGeneral = (json \ "attributes" \ "types" \ "resourceTypeGeneral").extractOrElse[String](null)
|
||||
val schemaOrg = (json \ "attributes" \ "types" \ "schemaOrg").extractOrElse[String](null)
|
||||
|
||||
val doi = (json \ "attributes" \ "doi").extract[String]
|
||||
if (doi.isEmpty)
|
||||
return List()
|
||||
|
||||
//Mapping type based on vocabularies dnet:publication_resource and dnet:result_typologies
|
||||
val result = getResult(resourceType, resourceTypeGeneral, schemaOrg, vocabularies)
|
||||
if (result == null)
|
||||
return List()
|
||||
|
||||
|
||||
val doi_q = OafMapperUtils.qualifier("doi", "doi", ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES)
|
||||
val pid = OafMapperUtils.structuredProperty(doi, doi_q, dataInfo)
|
||||
result.setPid(List(pid).asJava)
|
||||
result.setId(OafMapperUtils.createOpenaireId(50, s"datacite____::$doi", true))
|
||||
result.setOriginalId(List(doi).asJava)
|
||||
|
||||
val d = new Date(dateOfCollection * 1000)
|
||||
val ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US)
|
||||
|
||||
|
||||
result.setDateofcollection(ISO8601FORMAT.format(d))
|
||||
result.setDateoftransformation(ISO8601FORMAT.format(ts))
|
||||
result.setDataInfo(dataInfo)
|
||||
|
||||
val creators = (json \\ "creators").extractOrElse[List[CreatorType]](List())
|
||||
|
||||
|
||||
val authors = creators.zipWithIndex.map { case (c, idx) =>
|
||||
val a = new Author
|
||||
a.setFullname(c.name.orNull)
|
||||
a.setName(c.givenName.orNull)
|
||||
a.setSurname(c.familyName.orNull)
|
||||
if (c.nameIdentifiers != null && c.nameIdentifiers.isDefined && c.nameIdentifiers.get != null) {
|
||||
a.setPid(c.nameIdentifiers.get.map(ni => {
|
||||
val q = if (ni.nameIdentifierScheme.isDefined) vocabularies.getTermAsQualifier(ModelConstants.DNET_PID_TYPES, ni.nameIdentifierScheme.get.toLowerCase()) else null
|
||||
if (ni.nameIdentifier != null && ni.nameIdentifier.isDefined) {
|
||||
OafMapperUtils.structuredProperty(ni.nameIdentifier.get, q, dataInfo)
|
||||
}
|
||||
else
|
||||
null
|
||||
|
||||
}
|
||||
)
|
||||
.asJava)
|
||||
}
|
||||
if (c.affiliation.isDefined)
|
||||
a.setAffiliation(c.affiliation.get.filter(af => af.nonEmpty).map(af => OafMapperUtils.field(af, dataInfo)).asJava)
|
||||
a.setRank(idx + 1)
|
||||
a
|
||||
}
|
||||
|
||||
|
||||
val titles: List[TitleType] = (json \\ "titles").extractOrElse[List[TitleType]](List())
|
||||
|
||||
result.setTitle(titles.filter(t => t.title.nonEmpty).map(t => {
|
||||
if (t.titleType.isEmpty) {
|
||||
OafMapperUtils.structuredProperty(t.title.get, MAIN_TITLE_QUALIFIER, null)
|
||||
} else {
|
||||
OafMapperUtils.structuredProperty(t.title.get, t.titleType.get, t.titleType.get, DNET_DATACITE_TITLE, DNET_DATACITE_TITLE, null)
|
||||
}
|
||||
}).asJava)
|
||||
|
||||
if (authors == null || authors.isEmpty || !authors.exists(a => a != null))
|
||||
return List()
|
||||
result.setAuthor(authors.asJava)
|
||||
|
||||
val dates = (json \\ "dates").extract[List[DateType]]
|
||||
val publication_year = (json \\ "publicationYear").extractOrElse[String](null)
|
||||
|
||||
val i_date = dates
|
||||
.filter(d => d.date.isDefined && d.dateType.isDefined)
|
||||
.find(d => d.dateType.get.equalsIgnoreCase("issued"))
|
||||
.map(d => extract_date(d.date.get))
|
||||
val a_date: Option[String] = dates
|
||||
.filter(d => d.date.isDefined && d.dateType.isDefined && d.dateType.get.equalsIgnoreCase("available"))
|
||||
.map(d => extract_date(d.date.get))
|
||||
.find(d => d != null && d.isDefined)
|
||||
.map(d => d.get)
|
||||
|
||||
if (a_date.isDefined) {
|
||||
if(doi.startsWith("10.14457"))
|
||||
result.setEmbargoenddate(OafMapperUtils.field(fix_thai_date(a_date.get,"[yyyy-MM-dd]"), null))
|
||||
else
|
||||
result.setEmbargoenddate(OafMapperUtils.field(a_date.get, null))
|
||||
}
|
||||
if (i_date.isDefined && i_date.get.isDefined) {
|
||||
if(doi.startsWith("10.14457")) {
|
||||
result.setDateofacceptance(OafMapperUtils.field(fix_thai_date(i_date.get.get,"[yyyy-MM-dd]"), null))
|
||||
result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(fix_thai_date(i_date.get.get,"[yyyy-MM-dd]"), null))
|
||||
}
|
||||
else {
|
||||
result.setDateofacceptance(OafMapperUtils.field(i_date.get.get, null))
|
||||
result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(i_date.get.get, null))
|
||||
}
|
||||
}
|
||||
else if (publication_year != null) {
|
||||
if(doi.startsWith("10.14457")) {
|
||||
result.setDateofacceptance(OafMapperUtils.field(fix_thai_date(s"01-01-$publication_year","[dd-MM-yyyy]"), null))
|
||||
result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(fix_thai_date(s"01-01-$publication_year","[dd-MM-yyyy]"), null))
|
||||
|
||||
} else {
|
||||
result.setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null))
|
||||
result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
result.setRelevantdate(dates.filter(d => d.date.isDefined && d.dateType.isDefined)
|
||||
.map(d => (extract_date(d.date.get), d.dateType.get))
|
||||
.filter(d => d._1.isDefined)
|
||||
.map(d => (d._1.get, vocabularies.getTermAsQualifier(DNET_DATACITE_DATE, d._2.toLowerCase())))
|
||||
.filter(d => d._2 != null)
|
||||
.map(d => generateOAFDate(d._1, d._2)).asJava)
|
||||
|
||||
val subjects = (json \\ "subjects").extract[List[SubjectType]]
|
||||
|
||||
result.setSubject(subjects.filter(s => s.subject.nonEmpty)
|
||||
.map(s =>
|
||||
OafMapperUtils.structuredProperty(s.subject.get, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, null)
|
||||
).asJava)
|
||||
|
||||
|
||||
result.setCollectedfrom(List(DATACITE_COLLECTED_FROM).asJava)
|
||||
|
||||
val descriptions = (json \\ "descriptions").extract[List[DescriptionType]]
|
||||
|
||||
result.setDescription(
|
||||
descriptions
|
||||
.filter(d => d.description.isDefined).
|
||||
map(d =>
|
||||
OafMapperUtils.field(d.description.get, null)
|
||||
).filter(s => s != null).asJava)
|
||||
|
||||
|
||||
val publisher = (json \\ "publisher").extractOrElse[String](null)
|
||||
if (publisher != null)
|
||||
result.setPublisher(OafMapperUtils.field(publisher, null))
|
||||
|
||||
|
||||
val language: String = (json \\ "language").extractOrElse[String](null)
|
||||
|
||||
if (language != null)
|
||||
result.setLanguage(vocabularies.getSynonymAsQualifier(ModelConstants.DNET_LANGUAGES, language))
|
||||
|
||||
|
||||
val instance = result.getInstance().get(0)
|
||||
|
||||
val client = (json \ "relationships" \ "client" \\ "id").extractOpt[String]
|
||||
|
||||
val accessRights: List[String] = for {
|
||||
JObject(rightsList) <- json \\ "rightsList"
|
||||
JField("rightsUri", JString(rightsUri)) <- rightsList
|
||||
} yield rightsUri
|
||||
|
||||
val aRights: Option[Qualifier] = accessRights.map(r => {
|
||||
vocabularies.getSynonymAsQualifier(ModelConstants.DNET_ACCESS_MODES, r)
|
||||
}).find(q => q != null).map(q => {
|
||||
val a = new Qualifier
|
||||
a.setClassid(q.getClassid)
|
||||
a.setClassname(q.getClassname)
|
||||
a.setSchemeid(q.getSchemeid)
|
||||
a.setSchemename(q.getSchemename)
|
||||
a
|
||||
})
|
||||
|
||||
|
||||
val access_rights_qualifier = if (aRights.isDefined) aRights.get else OafMapperUtils.qualifier(ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE, ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)
|
||||
|
||||
if (client.isDefined) {
|
||||
val hb = hostedByMap.getOrElse(client.get.toUpperCase(), unknown_repository)
|
||||
instance.setHostedby(OafMapperUtils.keyValue(generateDSId(hb.openaire_id), hb.official_name))
|
||||
instance.setCollectedfrom(DATACITE_COLLECTED_FROM)
|
||||
instance.setUrl(List(s"https://dx.doi.org/$doi").asJava)
|
||||
instance.setAccessright(access_rights_qualifier)
|
||||
val license = accessRights
|
||||
.find(r => r.startsWith("http") && r.matches(".*(/licenses|/publicdomain|unlicense\\.org/|/legal-and-data-protection-notices|/download/license|/open-government-licence).*"))
|
||||
if (license.isDefined)
|
||||
instance.setLicense(OafMapperUtils.field(license.get, null))
|
||||
}
|
||||
|
||||
val awardUris: List[String] = for {
|
||||
JObject(fundingReferences) <- json \\ "fundingReferences"
|
||||
JField("awardUri", JString(awardUri)) <- fundingReferences
|
||||
} yield awardUri
|
||||
|
||||
val relations: List[Relation] = awardUris.flatMap(a => get_projectRelation(a, result.getId)).filter(r => r != null)
|
||||
fix_figshare(result)
|
||||
if (relations != null && relations.nonEmpty) {
|
||||
List(result) ::: relations
|
||||
}
|
||||
else
|
||||
List(result)
|
||||
}
|
||||
|
||||
def generateDataInfo(trust: String): DataInfo = {
|
||||
val di = new DataInfo
|
||||
di.setDeletedbyinference(false)
|
||||
di.setInferred(false)
|
||||
di.setInvisible(false)
|
||||
di.setTrust(trust)
|
||||
di.setProvenanceaction(PROVENANCE_ACTION_SET_QUALIFIER)
|
||||
di
|
||||
}
|
||||
|
||||
def generateDSId(input: String): String = {
|
||||
val b = StringUtils.substringBefore(input, "::")
|
||||
val a = StringUtils.substringAfter(input, "::")
|
||||
s"10|$b::${DHPUtils.md5(a)}"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package eu.dnetlib.dhp.actionmanager.datacite
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser
|
||||
import eu.dnetlib.dhp.schema.oaf.Oaf
|
||||
import org.apache.hadoop.io.Text
|
||||
import org.apache.hadoop.io.compress.GzipCodec
|
||||
import org.apache.hadoop.mapred.SequenceFileOutputFormat
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
||||
import scala.io.Source
|
||||
|
||||
object ExportActionSetJobNode {
|
||||
|
||||
val log: Logger = LoggerFactory.getLogger(ExportActionSetJobNode.getClass)
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val conf = new SparkConf
|
||||
val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/exportDataset_parameters.json")).mkString)
|
||||
parser.parseArgument(args)
|
||||
val master = parser.get("master")
|
||||
val sourcePath = parser.get("sourcePath")
|
||||
val targetPath = parser.get("targetPath")
|
||||
|
||||
val spark: SparkSession = SparkSession.builder().config(conf)
|
||||
.appName(ExportActionSetJobNode.getClass.getSimpleName)
|
||||
.master(master)
|
||||
.getOrCreate()
|
||||
implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
|
||||
implicit val tEncoder:Encoder[(String,String)] = Encoders.tuple(Encoders.STRING,Encoders.STRING)
|
||||
|
||||
spark.read.load(sourcePath).as[Oaf]
|
||||
.map(o =>DataciteToOAFTransformation.toActionSet(o))
|
||||
.filter(o => o!= null)
|
||||
.rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$targetPath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec])
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package eu.dnetlib.dhp.actionmanager.datacite
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup
|
||||
import eu.dnetlib.dhp.schema.oaf.Oaf
|
||||
import eu.dnetlib.dhp.utils.ISLookupClientFactory
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
||||
import scala.io.Source
|
||||
|
||||
object GenerateDataciteDatasetSpark {
|
||||
|
||||
val log: Logger = LoggerFactory.getLogger(GenerateDataciteDatasetSpark.getClass)
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val conf = new SparkConf
|
||||
val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json")).mkString)
|
||||
parser.parseArgument(args)
|
||||
val master = parser.get("master")
|
||||
val sourcePath = parser.get("sourcePath")
|
||||
val targetPath = parser.get("targetPath")
|
||||
val isLookupUrl: String = parser.get("isLookupUrl")
|
||||
log.info("isLookupUrl: {}", isLookupUrl)
|
||||
|
||||
val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
|
||||
val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
|
||||
val spark: SparkSession = SparkSession.builder().config(conf)
|
||||
.appName(GenerateDataciteDatasetSpark.getClass.getSimpleName)
|
||||
.master(master)
|
||||
.getOrCreate()
|
||||
|
||||
implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
|
||||
|
||||
import spark.implicits._
|
||||
|
||||
spark.read.load(sourcePath).as[DataciteType]
|
||||
.filter(d => d.isActive)
|
||||
.flatMap(d => DataciteToOAFTransformation.generateOAF(d.json, d.timestamp, d.timestamp, vocabularies))
|
||||
.filter(d => d != null)
|
||||
.write.mode(SaveMode.Overwrite).save(targetPath)
|
||||
}
|
||||
}
|
||||
|
|
@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
|
||||
import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.common.HdfsSupport;
|
||||
|
|
@ -33,7 +32,6 @@ public class PrepareProjects {
|
|||
private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class);
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
String jsonConfiguration = IOUtils
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
TUBYDI - Assistir Filmes e Series Online Grátis
|
||||
123Movies
|
||||
WATCH FULL MOVIE
|
||||
Movierulz
|
||||
Full Movie Online
|
||||
MOVIé WatcH
|
||||
The King of Staten Island 2020 Online For Free
|
||||
Watch Train to Busan 2 2020 online for free
|
||||
Sixth Sense Movie Novelization
|
||||
Film Complet streaming vf gratuit en ligne
|
||||
watch now free
|
||||
LIVE stream watch
|
||||
LIVE stream UFC
|
||||
RBC Heritage live stream
|
||||
MLBStreams Free
|
||||
NFL Live Stream
|
||||
Live Stream Free
|
||||
Royal Ascot 2020 Live Stream
|
||||
TV Shows Full Episodes Official
|
||||
FuboTV
|
||||
Gomovies
|
||||
Online Free Trial Access
|
||||
123watch
|
||||
DÜŞÜK HAPI
|
||||
Bebek Düşürme Yöntemleri
|
||||
WHATSAP İLETİŞİM
|
||||
Cytotec
|
||||
düşük hapı
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
[
|
||||
{
|
||||
"paramName": "s",
|
||||
"paramLongName": "sourcePath",
|
||||
"paramDescription": "the source mdstore path",
|
||||
"paramRequired": true
|
||||
},
|
||||
|
||||
{
|
||||
"paramName": "t",
|
||||
"paramLongName": "targetPath",
|
||||
"paramDescription": "the target mdstore path",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "m",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "the master name",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
[
|
||||
{
|
||||
"paramName": "s",
|
||||
"paramLongName": "sourcePath",
|
||||
"paramDescription": "the source mdstore path",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "t",
|
||||
"paramLongName": "targetPath",
|
||||
"paramDescription": "the target mdstore path",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "m",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "the master name",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "i",
|
||||
"paramLongName": "isLookupUrl",
|
||||
"paramDescription": "the isLookup URL",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,23 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>yarnRM</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>hdfs://nameservice1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
<workflow-app name="Import_Datacite_and_transform_to_OAF" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>mainPath</name>
|
||||
<description>the working path of Datacite stores</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>isLookupUrl</name>
|
||||
<description>The IS lookUp service endopoint</description>
|
||||
</property>
|
||||
</parameters>
|
||||
|
||||
<start to="TransformJob"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
<action name="TransformJob">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>TransformJob</name>
|
||||
<class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
|
||||
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
|
||||
<arg>--targetPath</arg><arg>${mainPath}/production/datacite_oaf</arg>
|
||||
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>yarnRM</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>hdfs://nameservice1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
<workflow-app name="Datacite_to_ActionSet_Workflow" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>sourcePath</name>
|
||||
<description>the working path of Datacite stores</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>outputPath</name>
|
||||
<description>the path of Datacite ActionSet</description>
|
||||
</property>
|
||||
</parameters>
|
||||
|
||||
<start to="ExportDataset"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
|
||||
<action name="ExportDataset">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>ExportDataset</name>
|
||||
<class>eu.dnetlib.dhp.actionmanager.datacite.ExportActionSetJobNode</class>
|
||||
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||
<arg>--targetPath</arg><arg>${outputPath}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
|
||||
package eu.dentlib.dhp.aggregation;
|
||||
|
||||
import static org.mockito.Mockito.lenient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.mockito.Mock;
|
||||
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
|
||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||
|
||||
public abstract class AbstractVocabularyTest {
|
||||
|
||||
@Mock
|
||||
protected ISLookUpService isLookUpService;
|
||||
|
||||
protected VocabularyGroup vocabularies;
|
||||
|
||||
public void setUpVocabulary() throws ISLookUpException, IOException {
|
||||
lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs());
|
||||
|
||||
lenient()
|
||||
.when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY))
|
||||
.thenReturn(synonyms());
|
||||
vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService);
|
||||
}
|
||||
|
||||
private static List<String> vocs() throws IOException {
|
||||
return IOUtils
|
||||
.readLines(
|
||||
Objects
|
||||
.requireNonNull(
|
||||
AbstractVocabularyTest.class.getResourceAsStream("/eu/dnetlib/dhp/vocabulary/terms.txt")));
|
||||
}
|
||||
|
||||
private static List<String> synonyms() throws IOException {
|
||||
return IOUtils
|
||||
.readLines(
|
||||
Objects
|
||||
.requireNonNull(
|
||||
AbstractVocabularyTest.class.getResourceAsStream("/eu/dnetlib/dhp/vocabulary/synonyms.txt")));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package eu.dnetlib.dhp.actionmanager.datacite
|
||||
|
||||
|
||||
import eu.dentlib.dhp.aggregation.AbstractVocabularyTest
|
||||
import eu.dnetlib.dhp.schema.oaf.Oaf
|
||||
import org.junit.jupiter.api.extension.ExtendWith
|
||||
import org.junit.jupiter.api.{BeforeEach, Test}
|
||||
import org.mockito.junit.jupiter.MockitoExtension
|
||||
import org.codehaus.jackson.map.ObjectMapper
|
||||
|
||||
import scala.io.Source
|
||||
|
||||
@ExtendWith(Array(classOf[MockitoExtension]))
|
||||
class DataciteToOAFTest extends AbstractVocabularyTest{
|
||||
|
||||
|
||||
@BeforeEach
|
||||
def setUp() :Unit = {
|
||||
|
||||
super.setUpVocabulary()
|
||||
}
|
||||
|
||||
@Test
|
||||
def testMapping() :Unit = {
|
||||
val record =Source.fromInputStream(getClass.getResourceAsStream("datacite.json")).mkString
|
||||
|
||||
|
||||
|
||||
val mapper = new ObjectMapper()
|
||||
val res:List[Oaf] =DataciteToOAFTransformation.generateOAF(record, 0L,0L, vocabularies )
|
||||
println (mapper.defaultPrettyPrintingWriter().writeValueAsString(res.head))
|
||||
|
||||
|
||||
}
|
||||
@Test
|
||||
def testDate():Unit = {
|
||||
|
||||
println(DataciteToOAFTransformation.fix_thai_date("01-01-2561","[dd-MM-yyyy]"))
|
||||
println(DataciteToOAFTransformation.fix_thai_date("2561-01-01","[yyyy-MM-dd]"))
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -26,6 +26,7 @@ import eu.dnetlib.broker.objects.OaBrokerRelatedDatasource;
|
|||
import eu.dnetlib.broker.objects.OaBrokerRelatedPublication;
|
||||
import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware;
|
||||
import eu.dnetlib.broker.objects.OaBrokerTypedValue;
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||
import eu.dnetlib.dhp.schema.oaf.Datasource;
|
||||
|
|
@ -144,7 +145,7 @@ public class ConversionUtils {
|
|||
.filter(pid -> pid != null)
|
||||
.filter(pid -> pid.getQualifier() != null)
|
||||
.filter(pid -> pid.getQualifier().getClassid() != null)
|
||||
.filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid"))
|
||||
.filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase(ModelConstants.ORCID))
|
||||
.map(pid -> pid.getValue())
|
||||
.map(pid -> cleanOrcid(pid))
|
||||
.filter(StringUtils::isNotBlank)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import eu.dnetlib.dhp.schema.oaf.Field;
|
|||
|
||||
public class DatePicker {
|
||||
|
||||
private static final String DATE_PATTERN = "\\d{4}-\\d{2}-\\d{2}";
|
||||
private static final String DATE_PATTERN = "^(\\d{4})-(\\d{2})-(\\d{2})";
|
||||
private static final String DATE_DEFAULT_SUFFIX = "01-01";
|
||||
private static final int YEAR_LB = 1300;
|
||||
private static final int YEAR_UB = Year.now().getValue() + 5;
|
||||
|
|
@ -28,6 +28,7 @@ public class DatePicker {
|
|||
final Map<String, Integer> frequencies = dateofacceptance
|
||||
.parallelStream()
|
||||
.filter(StringUtils::isNotBlank)
|
||||
.map(d -> substringBefore(d, "T"))
|
||||
.collect(Collectors.toConcurrentMap(w -> w, w -> 1, Integer::sum));
|
||||
|
||||
if (frequencies.isEmpty()) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.dedup;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.clearspring.analytics.util.Lists;
|
||||
|
||||
public class DatePickerTest {
|
||||
|
||||
Collection<String> dates = Lists.newArrayList();
|
||||
|
||||
@Test
|
||||
public void testPickISO() {
|
||||
dates.add("2016-01-01T12:00:00Z");
|
||||
dates.add("2016-06-16T12:00:00Z");
|
||||
dates.add("2020-01-01T12:00:00Z");
|
||||
dates.add("2020-10-01T12:00:00Z");
|
||||
assertEquals("2020-10-01", DatePicker.pick(dates).getValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPickSimple() {
|
||||
dates.add("2016-01-01");
|
||||
dates.add("2016-06-16");
|
||||
dates.add("2020-01-01");
|
||||
dates.add("2020-10-01");
|
||||
assertEquals("2020-10-01", DatePicker.pick(dates).getValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPickFrequent() {
|
||||
dates.add("2016-02-01");
|
||||
dates.add("2016-02-01");
|
||||
dates.add("2016-02-01");
|
||||
dates.add("2020-10-01");
|
||||
assertEquals("2016-02-01", DatePicker.pick(dates).getValue());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import eu.dnetlib.dhp.schema.oaf.{DataInfo, Dataset, Field, Instance, KeyValue,
|
|||
import eu.dnetlib.dhp.utils.DHPUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants
|
||||
import org.json4s
|
||||
import org.json4s.DefaultFormats
|
||||
import org.json4s.jackson.JsonMethods.parse
|
||||
|
|
@ -28,7 +29,6 @@ object DoiBoostMappingUtil {
|
|||
//STATIC STRING
|
||||
val MAG = "microsoft"
|
||||
val MAG_NAME = "Microsoft Academic Graph"
|
||||
val ORCID = "orcid"
|
||||
val ORCID_PENDING = "orcid_pending"
|
||||
val CROSSREF = "Crossref"
|
||||
val UNPAYWALL = "UnpayWall"
|
||||
|
|
@ -37,8 +37,6 @@ object DoiBoostMappingUtil {
|
|||
val doiBoostNSPREFIX = "doiboost____"
|
||||
val OPENAIRE_PREFIX = "openaire____"
|
||||
val SEPARATOR = "::"
|
||||
val DNET_LANGUAGES = "dnet:languages"
|
||||
val PID_TYPES = "dnet:pid_types"
|
||||
|
||||
val invalidName = List(",", "none none", "none, none", "none &na;", "(:null)", "test test test", "test test", "test", "&na; &na;")
|
||||
|
||||
|
|
@ -326,8 +324,8 @@ object DoiBoostMappingUtil {
|
|||
def createORIDCollectedFrom(): KeyValue = {
|
||||
|
||||
val cf = new KeyValue
|
||||
cf.setValue(ORCID)
|
||||
cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ORCID.toLowerCase))
|
||||
cf.setValue(ModelConstants.ORCID_DS)
|
||||
cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ModelConstants.ORCID))
|
||||
cf
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ case object Crossref2Oaf {
|
|||
|
||||
//MAPPING Crossref DOI into PID
|
||||
val doi: String = (json \ "DOI").extract[String]
|
||||
result.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava)
|
||||
result.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
|
||||
|
||||
//MAPPING Crossref DOI into OriginalId
|
||||
//and Other Original Identifier of dataset like clinical-trial-number
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ object SparkMapDumpIntoOAF {
|
|||
implicit val mapEncoderPubs: Encoder[Publication] = Encoders.kryo[Publication]
|
||||
implicit val mapEncoderRelatons: Encoder[Relation] = Encoders.kryo[Relation]
|
||||
implicit val mapEncoderDatasets: Encoder[oaf.Dataset] = Encoders.kryo[OafDataset]
|
||||
implicit val mrEncoder: Encoder[CrossrefDT] = Encoders.kryo[CrossrefDT]
|
||||
|
||||
val targetPath = parser.get("targetPath")
|
||||
import spark.implicits._
|
||||
|
||||
spark.read.load(parser.get("sourcePath")).as[CrossrefDT]
|
||||
.flatMap(k => Crossref2Oaf.convert(k.json))
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ case object ConversionUtil {
|
|||
val authors = inputParams._2
|
||||
|
||||
val pub = new Publication
|
||||
pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava)
|
||||
pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
|
||||
pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava)
|
||||
|
||||
//Set identifier as 50|doiboost____::md5(DOI)
|
||||
|
|
@ -247,7 +247,7 @@ case object ConversionUtil {
|
|||
val description = inputParams._2
|
||||
|
||||
val pub = new Publication
|
||||
pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava)
|
||||
pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
|
||||
pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava)
|
||||
|
||||
//Set identifier as 50 | doiboost____::md5(DOI)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ public class PublicationToOaf implements Serializable {
|
|||
|
||||
static Logger logger = LoggerFactory.getLogger(PublicationToOaf.class);
|
||||
|
||||
public static final String ORCID = "ORCID";
|
||||
public final static String orcidPREFIX = "orcid_______";
|
||||
public static final String OPENAIRE_PREFIX = "openaire____";
|
||||
public static final String SEPARATOR = "::";
|
||||
|
|
@ -69,7 +68,9 @@ public class PublicationToOaf implements Serializable {
|
|||
private static Map<String, Pair<String, String>> datasources = new HashMap<String, Pair<String, String>>() {
|
||||
|
||||
{
|
||||
put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + "orcid"));
|
||||
put(
|
||||
ModelConstants.ORCID,
|
||||
new Pair<>(ModelConstants.ORCID_DS, OPENAIRE_PREFIX + SEPARATOR + ModelConstants.ORCID));
|
||||
|
||||
}
|
||||
};
|
||||
|
|
@ -102,8 +103,6 @@ public class PublicationToOaf implements Serializable {
|
|||
}
|
||||
}
|
||||
|
||||
public static final String PID_TYPES = "dnet:pid_types";
|
||||
|
||||
public Oaf generatePublicationActionsFromJson(final String json) {
|
||||
try {
|
||||
if (parsedPublications != null) {
|
||||
|
|
@ -138,8 +137,8 @@ public class PublicationToOaf implements Serializable {
|
|||
mapQualifier(
|
||||
"sysimport:actionset:orcidworks-no-doi",
|
||||
"sysimport:actionset:orcidworks-no-doi",
|
||||
"dnet:provenanceActions",
|
||||
"dnet:provenanceActions"));
|
||||
ModelConstants.DNET_PROVENANCE_ACTIONS,
|
||||
ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||
publication.setDataInfo(dataInfo);
|
||||
|
||||
publication.setLastupdatetimestamp(new Date().getTime());
|
||||
|
|
@ -159,7 +158,9 @@ public class PublicationToOaf implements Serializable {
|
|||
publication
|
||||
.getExternalReference()
|
||||
.add(
|
||||
convertExtRef(extId, classid, classname, "dnet:pid_types", "dnet:pid_types"));
|
||||
convertExtRef(
|
||||
extId, classid, classname, ModelConstants.DNET_PID_TYPES,
|
||||
ModelConstants.DNET_PID_TYPES));
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -505,24 +506,21 @@ public class PublicationToOaf implements Serializable {
|
|||
|
||||
private KeyValue createCollectedFrom() {
|
||||
KeyValue cf = new KeyValue();
|
||||
cf.setValue(ORCID);
|
||||
cf.setValue(ModelConstants.ORCID_DS);
|
||||
cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "806360c771262b4d6770e7cdf04b5c5a");
|
||||
return cf;
|
||||
}
|
||||
|
||||
private KeyValue createHostedBy() {
|
||||
KeyValue hb = new KeyValue();
|
||||
hb.setValue("Unknown Repository");
|
||||
hb.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "55045bd2a65019fd8e6741a755395c8c");
|
||||
return hb;
|
||||
return ModelConstants.UNKNOWN_REPOSITORY;
|
||||
}
|
||||
|
||||
private StructuredProperty mapAuthorId(String orcidId) {
|
||||
final StructuredProperty sp = new StructuredProperty();
|
||||
sp.setValue(orcidId);
|
||||
final Qualifier q = new Qualifier();
|
||||
q.setClassid(ORCID.toLowerCase());
|
||||
q.setClassname(ORCID.toLowerCase());
|
||||
q.setClassid(ModelConstants.ORCID);
|
||||
q.setClassname(ModelConstants.ORCID_CLASSNAME);
|
||||
q.setSchemeid(ModelConstants.DNET_PID_TYPES);
|
||||
q.setSchemename(ModelConstants.DNET_PID_TYPES);
|
||||
sp.setQualifier(q);
|
||||
|
|
@ -535,8 +533,8 @@ public class PublicationToOaf implements Serializable {
|
|||
mapQualifier(
|
||||
"sysimport:crosswalk:entityregistry",
|
||||
"Harvested",
|
||||
"dnet:provenanceActions",
|
||||
"dnet:provenanceActions"));
|
||||
ModelConstants.DNET_PROVENANCE_ACTIONS,
|
||||
ModelConstants.DNET_PROVENANCE_ACTIONS));
|
||||
sp.setDataInfo(dataInfo);
|
||||
return sp;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
package eu.dnetlib.doiboost.uw
|
||||
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants
|
||||
import eu.dnetlib.dhp.schema.oaf.{Instance, Publication}
|
||||
import org.json4s
|
||||
import org.json4s.DefaultFormats
|
||||
|
|
@ -32,7 +33,7 @@ object UnpayWallToOAF {
|
|||
val is_oa = (json\ "is_oa").extract[Boolean]
|
||||
|
||||
val oaLocation:OALocation = (json \ "best_oa_location").extractOrElse[OALocation](null)
|
||||
pub.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava)
|
||||
pub.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
|
||||
pub.setId(generateIdentifier(pub, doi.toLowerCase))
|
||||
|
||||
pub.setCollectedfrom(List(createUnpayWallCollectedFrom()).asJava)
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>yarnRM</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>hdfs://nameservice1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>hive_metastore_uris</name>
|
||||
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2YarnHistoryServerAddress</name>
|
||||
<value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2EventLogDir</name>
|
||||
<value>/user/spark/spark2ApplicationHistory</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2ExtraListeners</name>
|
||||
<value>"com.cloudera.spark.lineage.NavigatorAppListener"</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2SqlQueryExecutionListeners</name>
|
||||
<value>"com.cloudera.spark.lineage.NavigatorQueryListener"</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -1,372 +0,0 @@
|
|||
<workflow-app name="Generate DOIBoost ActionSet" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>sparkDriverMemory</name>
|
||||
<description>memory for driver process</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sparkExecutorMemory</name>
|
||||
<description>memory for individual executor</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>sparkExecutorIntersectionMemory</name>
|
||||
<description>memory for individual executor</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>sparkExecutorCores</name>
|
||||
<description>number of cores used by single executor</description>
|
||||
</property>
|
||||
|
||||
|
||||
<!-- Itersection Parameters -->
|
||||
<property>
|
||||
<name>workingPath</name>
|
||||
<description>the working Path</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>hostedByMapPath</name>
|
||||
<description>the hostedByMap Path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>outputPath</name>
|
||||
<description>the Path of the sequence file action set</description>
|
||||
</property>
|
||||
|
||||
|
||||
<!-- Crossref Parameters -->
|
||||
<property>
|
||||
<name>inputPathCrossref</name>
|
||||
<description>the Crossref input path</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>crossrefTimestamp</name>
|
||||
<description>Timestamp for the Crossref incremental Harvesting</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>esServer</name>
|
||||
<description>elasticsearch server url for the Crossref Harvesting</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>esIndex</name>
|
||||
<description>elasticsearch index name for the Crossref Harvesting</description>
|
||||
</property>
|
||||
|
||||
<!-- MAG Parameters -->
|
||||
<property>
|
||||
<name>MAGDumpPath</name>
|
||||
<description>the MAG dump working path</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>inputPathMAG</name>
|
||||
<description>the MAG working path</description>
|
||||
</property>
|
||||
|
||||
|
||||
<!-- UnpayWall Parameters -->
|
||||
<property>
|
||||
<name>inputPathUnpayWall</name>
|
||||
<description>the UnpayWall working path</description>
|
||||
</property>
|
||||
|
||||
<!-- ORCID Parameters -->
|
||||
<property>
|
||||
<name>inputPathOrcid</name>
|
||||
<description>the ORCID input path</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>workingPathOrcid</name>
|
||||
<description>the ORCID working path</description>
|
||||
</property>
|
||||
|
||||
</parameters>
|
||||
|
||||
<global>
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>${oozieActionShareLibForSpark2}</value>
|
||||
</property>
|
||||
</configuration>
|
||||
</global>
|
||||
|
||||
<start to="resume_from"/>
|
||||
|
||||
<decision name="resume_from">
|
||||
<switch>
|
||||
<case to="ConvertCrossrefToOAF">${wf:conf('resumeFrom') eq 'ConvertCrossrefToOAF'}</case>
|
||||
<case to="ResetMagWorkingPath">${wf:conf('resumeFrom') eq 'ResetMagWorkingPath'}</case>
|
||||
<case to="ProcessMAG">${wf:conf('resumeFrom') eq 'PreprocessMag'}</case>
|
||||
<case to="ProcessUW">${wf:conf('resumeFrom') eq 'PreprocessUW'}</case>
|
||||
<case to="ProcessORCID">${wf:conf('resumeFrom') eq 'PreprocessORCID'}</case>
|
||||
<case to="CreateDOIBoost">${wf:conf('resumeFrom') eq 'CreateDOIBoost'}</case>
|
||||
<case to="GenerateActionSet">${wf:conf('resumeFrom') eq 'GenerateActionSet'}</case>
|
||||
<default to="ImportCrossRef"/>
|
||||
</switch>
|
||||
</decision>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
<action name="ImportCrossRef">
|
||||
<java>
|
||||
<main-class>eu.dnetlib.doiboost.crossref.CrossrefImporter</main-class>
|
||||
<arg>--targetPath</arg><arg>${inputPathCrossref}/index_update</arg>
|
||||
<arg>--namenode</arg><arg>${nameNode}</arg>
|
||||
<arg>--esServer</arg><arg>${esServer}</arg>
|
||||
<arg>--esIndex</arg><arg>${esIndex}</arg>
|
||||
<arg>--timestamp</arg><arg>${crossrefTimestamp}</arg>
|
||||
</java>
|
||||
<ok to="GenerateCrossrefDataset"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
<!-- CROSSREF SECTION -->
|
||||
|
||||
<action name="GenerateCrossrefDataset">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>GenerateCrossrefDataset</name>
|
||||
<class>eu.dnetlib.doiboost.crossref.CrossrefDataset</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--workingPath</arg><arg>${inputPathCrossref}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="RenameDataset"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="RenameDataset">
|
||||
<fs>
|
||||
<delete path="${inputPathCrossref}/crossref_ds"/>
|
||||
<move source="${inputPathCrossref}/crossref_ds_updated"
|
||||
target="${inputPathCrossref}/crossref_ds"/>
|
||||
</fs>
|
||||
<ok to="ResetMagWorkingPath"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
|
||||
<!-- MAG SECTION -->
|
||||
<action name="ResetMagWorkingPath">
|
||||
<fs>
|
||||
<delete path="${inputPathMAG}/dataset"/>
|
||||
<delete path="${inputPathMAG}/process"/>
|
||||
</fs>
|
||||
<ok to="ConvertMagToDataset"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="ConvertMagToDataset">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Convert Mag to Dataset</name>
|
||||
<class>eu.dnetlib.doiboost.mag.SparkImportMagIntoDataset</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${MAGDumpPath}</arg>
|
||||
<arg>--targetPath</arg><arg>${inputPathMAG}/dataset</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="ConvertCrossrefToOAF"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
<action name="ConvertCrossrefToOAF">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>ConvertCrossrefToOAF</name>
|
||||
<class>eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${inputPathCrossref}/crossref_ds</arg>
|
||||
<arg>--targetPath</arg><arg>${workingPath}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="ProcessMAG"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="ProcessMAG">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Convert Mag to OAF Dataset</name>
|
||||
<class>eu.dnetlib.doiboost.mag.SparkProcessMAG</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorIntersectionMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${inputPathMAG}/dataset</arg>
|
||||
<arg>--workingPath</arg><arg>${inputPathMAG}/process</arg>
|
||||
<arg>--targetPath</arg><arg>${workingPath}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="ProcessUW"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<!-- UnpayWall SECTION -->
|
||||
|
||||
<action name="ProcessUW">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Convert UnpayWall to Dataset</name>
|
||||
<class>eu.dnetlib.doiboost.uw.SparkMapUnpayWallToOAF</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${inputPathUnpayWall}/uw_extracted</arg>
|
||||
<arg>--targetPath</arg><arg>${workingPath}/uwPublication</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="ProcessORCID"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<!-- ORCID SECTION -->
|
||||
<action name="ProcessORCID">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Convert ORCID to Dataset</name>
|
||||
<class>eu.dnetlib.doiboost.orcid.SparkConvertORCIDToOAF</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${inputPathOrcid}</arg>
|
||||
<arg>--workingPath</arg><arg>${workingPathOrcid}</arg>
|
||||
<arg>--targetPath</arg><arg>${workingPath}/orcidPublication</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="CreateDOIBoost"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<!-- INTERSECTION SECTION-->
|
||||
<action name="CreateDOIBoost">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Create DOIBoost Infospace</name>
|
||||
<class>eu.dnetlib.doiboost.SparkGenerateDoiBoost</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorIntersectionMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--hostedByMapPath</arg><arg>${hostedByMapPath}</arg>
|
||||
<arg>--affiliationPath</arg><arg>${inputPathMAG}/dataset/Affiliations</arg>
|
||||
<arg>--paperAffiliationPath</arg><arg>${inputPathMAG}/dataset/PaperAuthorAffiliations</arg>
|
||||
<arg>--workingPath</arg><arg>${workingPath}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="GenerateActionSet"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
<action name="GenerateActionSet">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn-cluster</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Generate DOIBoost ActionSet</name>
|
||||
<class>eu.dnetlib.doiboost.SparkGenerateDOIBoostActionSet</class>
|
||||
<jar>dhp-doiboost-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.sql.shuffle.partitions=3840
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--dbPublicationPath</arg><arg>${workingPath}/doiBoostPublicationFiltered</arg>
|
||||
<arg>--dbDatasetPath</arg><arg>${workingPath}/crossrefDataset</arg>
|
||||
<arg>--crossRefRelation</arg><arg>${workingPath}/crossrefRelation</arg>
|
||||
<arg>--dbaffiliationRelationPath</arg><arg>${workingPath}/doiBoostPublicationAffiliation</arg>
|
||||
<arg>--dbOrganizationPath</arg><arg>${workingPath}/doiBoostOrganization</arg>
|
||||
<arg>--targetPath</arg><arg>${workingPath}/actionDataSet</arg>
|
||||
<arg>--sFilePath</arg><arg>${outputPath}</arg>
|
||||
<arg>--master</arg><arg>yarn-cluster</arg>
|
||||
</spark>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -24,8 +24,6 @@ public class Constants {
|
|||
|
||||
public static String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
|
||||
|
||||
public static String ORCID = "orcid";
|
||||
|
||||
static {
|
||||
accessRightsCoarMap.put("OPEN", "c_abf2");
|
||||
accessRightsCoarMap.put("RESTRICTED", "c_16ec");
|
||||
|
|
|
|||
|
|
@ -503,7 +503,7 @@ public class ResultMapper implements Serializable {
|
|||
|
||||
private static Pid getOrcid(List<StructuredProperty> p) {
|
||||
for (StructuredProperty pid : p) {
|
||||
if (pid.getQualifier().getClassid().equals(Constants.ORCID)) {
|
||||
if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
|
||||
Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
|
||||
if (di.isPresent()) {
|
||||
return Pid
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ public abstract class AbstractMdRecordToOafMapper {
|
|||
protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3";
|
||||
protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/";
|
||||
protected static final Qualifier ORCID_PID_TYPE = qualifier(
|
||||
"ORCID", "Open Researcher and Contributor ID", DNET_PID_TYPES, DNET_PID_TYPES);
|
||||
ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, DNET_PID_TYPES, DNET_PID_TYPES);
|
||||
protected static final Qualifier MAG_PID_TYPE = qualifier(
|
||||
"MAGIdentifier", "Microsoft Academic Graph Identifier", DNET_PID_TYPES, DNET_PID_TYPES);
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import com.google.common.collect.Lists;
|
|||
|
||||
import eu.dnetlib.dhp.common.PacePerson;
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
||||
import eu.dnetlib.dhp.schema.oaf.Field;
|
||||
|
|
@ -61,7 +62,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper {
|
|||
author.setPid(new ArrayList<>());
|
||||
|
||||
if (StringUtils.isNotBlank(pid)) {
|
||||
if (type.startsWith("ORCID")) {
|
||||
if (type.toLowerCase().startsWith(ORCID)) {
|
||||
final String cleanedId = pid
|
||||
.replaceAll("http://orcid.org/", "")
|
||||
.replaceAll("https://orcid.org/", "");
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
|
|||
|
||||
import eu.dnetlib.dhp.common.PacePerson;
|
||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
|
||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||
import eu.dnetlib.dhp.schema.oaf.DataInfo;
|
||||
import eu.dnetlib.dhp.schema.oaf.Field;
|
||||
|
|
@ -98,7 +99,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
|
|||
.replaceAll(" ", "")
|
||||
.replaceAll("_", "");
|
||||
|
||||
if (type.startsWith("ORCID")) {
|
||||
if (type.toLowerCase().startsWith(ModelConstants.ORCID)) {
|
||||
final String cleanedId = id.replaceAll("http://orcid.org/", "").replaceAll("https://orcid.org/", "");
|
||||
res.add(structuredProperty(cleanedId, ORCID_PID_TYPE, info));
|
||||
} else if (type.startsWith("MAGID")) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project-shared-configuration>
|
||||
<!--
|
||||
This file contains additional configuration written by modules in the NetBeans IDE.
|
||||
The configuration is intended to be shared among all the users of project and
|
||||
therefore it is assumed to be part of version control checkout.
|
||||
Without this configuration present, some functionality in the IDE may be limited or fail altogether.
|
||||
-->
|
||||
<properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
|
||||
<!--
|
||||
Properties that influence various parts of the IDE, especially code formatting and the like.
|
||||
You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
|
||||
That way multiple projects can share the same settings (useful for formatting rules for example).
|
||||
Any value defined here will override the pom.xml file value but is only applicable to the current project.
|
||||
-->
|
||||
<netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
|
||||
</properties>
|
||||
</project-shared-configuration>
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
|
||||
<parent>
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<version>1.1.7-SNAPSHOT</version>
|
||||
<relativePath>../</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dhp-indicators</artifactId>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>pl.project13.maven</groupId>
|
||||
<artifactId>git-commit-id-plugin</artifactId>
|
||||
<version>2.1.15</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>revision</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
|
||||
<!-- more config here as you see fit -->
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.6.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<cdh.hive.version>0.13.1-cdh5.2.1</cdh.hive.version>
|
||||
<cdh.hadoop.version>2.5.0-cdh5.2.1</cdh.hadoop.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.11</artifactId>
|
||||
<version>2.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.11</artifactId>
|
||||
<version>2.4.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.googlecode.json-simple</groupId>
|
||||
<artifactId>json-simple</artifactId>
|
||||
<version>1.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20180130</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-jdbc</artifactId>
|
||||
<version>${cdh.hive.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>2.7.4</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-common</artifactId>
|
||||
<version>1.1.7-SNAPSHOT</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mchange</groupId>
|
||||
<artifactId>c3p0</artifactId>
|
||||
<version>0.9.5.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>c3p0</groupId>
|
||||
<artifactId>c3p0</artifactId>
|
||||
<version>0.9.1.2</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.26</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<name>dhp-indicators</name>
|
||||
</project>
|
||||
|
|
@ -0,0 +1 @@
|
|||
mvn clean package -Poozie-package,deploy,run -Dworkflow.source.dir=eu/dnetlib/dhp/oa/graph/indicators
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>${jobTracker}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>${nameNode}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>hive_metastore_uris</name>
|
||||
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>hive_jdbc_url</name>
|
||||
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.wf.workflow.notification.url</name>
|
||||
<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
|
||||
</property>
|
||||
<!-- <property>
|
||||
<name>stats_tool_api_url</name>
|
||||
<value>${stats_tool_api_url}</value>
|
||||
</property>-->
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
create table TARGET.indi_pub_green_oa stored as parquet as
|
||||
select distinct p.id, coalesce(green_oa, 0) as green_oa
|
||||
from SOURCE.publication p
|
||||
left outer join (
|
||||
select p.id, 1 as green_oa
|
||||
from SOURCE.publication p
|
||||
join SOURCE.result_instance ri on ri.id = p.id
|
||||
join SOURCE.datasource on datasource.id = ri.hostedby
|
||||
where SOURCE.datasource.type like '%Repository%'
|
||||
and (ri.accessright = 'Open Access'
|
||||
or ri.accessright = 'Embargo')) tmp
|
||||
on p.id= tmp.id;
|
||||
|
||||
create table TARGET.indi_pub_grey_lit stored as parquet as
|
||||
select distinct p.id, coalesce(grey_lit, 0) as grey_lit
|
||||
from SOURCE.publication p
|
||||
left outer join (
|
||||
select p.id, 1 as grey_lit
|
||||
from SOURCE.publication p
|
||||
join SOURCE.result_classifications rt on rt.id = p.id
|
||||
where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and
|
||||
not exists (select 1 from SOURCE.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id;
|
||||
|
||||
create table TARGET.indi_pub_doi_from_crossref stored as parquet as
|
||||
select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref
|
||||
from SOURCE.publication p
|
||||
left outer join
|
||||
(select ri.id, 1 as doi_from_crossref from SOURCE.result_instance ri
|
||||
join SOURCE.datasource d on d.id = ri.collectedfrom
|
||||
where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp
|
||||
on tmp.id=p.id;
|
||||
|
||||
create table TARGET.indi_pub_gold_oa stored as parquet as
|
||||
select distinct p.id, coalesce(gold_oa, 0) as gold_oa
|
||||
from SOURCE.publication p
|
||||
left outer join (
|
||||
select p.id, 1 as gold_oa
|
||||
from SOURCE.publication p
|
||||
join SOURCE.result_instance ri on ri.id = p.id
|
||||
join SOURCE.datasource on datasource.id = ri.hostedby
|
||||
where SOURCE.datasource.id like '%doajarticles%') tmp
|
||||
on p.id= tmp.id;
|
||||
|
||||
compute stats TARGET.indi_pub_green_oa;
|
||||
compute stats TARGET.indi_pub_grey_lit;
|
||||
compute stats TARGET.indi_pub_doi_from_crossref;
|
||||
compute stats TARGET.indi_pub_gold_oa;
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
export SOURCE=$1
|
||||
export TARGET=$2
|
||||
export SHADOW=$3
|
||||
export SCRIPT_PATH=$4
|
||||
|
||||
echo "Getting file from " $4
|
||||
hdfs dfs -copyToLocal $4
|
||||
|
||||
echo "Creating indicators database"
|
||||
impala-shell -q "drop database if exists ${TARGET} cascade"
|
||||
impala-shell -q "create database if not exists ${TARGET}"
|
||||
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${TARGET}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -f -
|
||||
cat createIndicatorsTables.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f -
|
||||
echo "Indicators Database created"
|
||||
|
||||
|
||||
echo "Updating Shadow indicators DB"
|
||||
impala-shell -q "create database if not exists ${SHADOW}"
|
||||
impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
|
||||
impala-shell -d ${TARGET} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${TARGET}.\1;/" | impala-shell -f -
|
||||
echo "Indicators Shadow DB ready!"
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
<workflow-app name="Indicators" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>stats_db_name</name>
|
||||
<description>the source stats database name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>indicators_db_name</name>
|
||||
<description>the target indicators database name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>indicators_shadow_db_name</name>
|
||||
<description>the name of the shadow schema</description>
|
||||
</property>
|
||||
<!-- <property>
|
||||
<name>openaire_db_name</name>
|
||||
<description>the original graph database name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>external_stats_db_name</name>
|
||||
<value>stats_ext</value>
|
||||
<description>the external stats that should be added since they are not included in the graph database</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>stats_db_shadow_name</name>
|
||||
<description>the name of the shadow schema</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>monitor_db_name</name>
|
||||
<description>the target monitor db name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>monitor_db_shadow_name</name>
|
||||
<description>the name of the shadow monitor db</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_name</name>
|
||||
<description>the target monitor db name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_shadow_name</name>
|
||||
<description>the name of the shadow monitor db</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>stats_tool_api_url</name>
|
||||
<description>The url of the API of the stats tool. Is used to trigger the cache update.</description>
|
||||
</property>-->
|
||||
<property>
|
||||
<name>hive_metastore_uris</name>
|
||||
<description>hive server metastore URIs</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hive_jdbc_url</name>
|
||||
<description>hive server jdbc url</description>
|
||||
</property>
|
||||
<!-- <property>
|
||||
<name>hive_timeout</name>
|
||||
<description>the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds.</description>
|
||||
</property>-->
|
||||
<!-- <property>
|
||||
<name>context_api_url</name>
|
||||
<description>the base url of the context api (https://services.openaire.eu/openaire)</description>
|
||||
</property>-->
|
||||
</parameters>
|
||||
|
||||
<global>
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>hive.metastore.uris</name>
|
||||
<value>${hive_metastore_uris}</value>
|
||||
</property>
|
||||
<!-- <property>
|
||||
<name>hive.txn.timeout</name>
|
||||
<value>${hive_timeout}</value>
|
||||
</property>-->
|
||||
</configuration>
|
||||
</global>
|
||||
|
||||
<start to="Step1-createIndicatorsDB"/>
|
||||
<action name="Step1-createIndicatorsDB">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>indicators.sh</exec>
|
||||
<argument>${stats_db_name}</argument>
|
||||
<argument>${indicators_db_name}</argument>
|
||||
<argument>${indicators_shadow_db_name}</argument>
|
||||
<argument>${wf:appPath()}/scripts/createIndicatorsTables.sql</argument>
|
||||
<file>scripts/indicators.sh</file>
|
||||
</shell>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
@ -16,6 +16,14 @@
|
|||
<name>monitor_db_production_name</name>
|
||||
<description>the name of the monitor public database</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_name</name>
|
||||
<description>the monitor database name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_production_name</name>
|
||||
<description>the name of the monitor public database</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>stats_tool_api_url</name>
|
||||
<description>The url of the API of the stats tool. Is used to trigger the cache promote.</description>
|
||||
|
|
@ -77,6 +85,19 @@
|
|||
<argument>${monitor_db_production_name}</argument>
|
||||
<file>updateProductionViews.sh</file>
|
||||
</shell>
|
||||
<ok to="updateObservatoryViews"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="updateObservatoryViews">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>updateProductionViews.sh</exec>
|
||||
<argument>${observatory_db_name}</argument>
|
||||
<argument>${observatory_db_production_name}</argument>
|
||||
<file>updateProductionViews.sh</file>
|
||||
</shell>
|
||||
<ok to="promoteCache"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
export SOURCE=$1
|
||||
export TARGET=$2
|
||||
export SHADOW=$3
|
||||
export SCRIPT_PATH=$4
|
||||
|
||||
echo "Getting file from " $4
|
||||
hdfs dfs -copyToLocal $4
|
||||
|
||||
echo "Creating observatory database"
|
||||
impala-shell -q "drop database if exists ${TARGET} cascade"
|
||||
impala-shell -q "create database if not exists ${TARGET}"
|
||||
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${TARGET}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -f -
|
||||
cat step21-createObservatoryDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f -
|
||||
echo "Impala shell finished"
|
||||
|
||||
echo "Updating shadow observatory database"
|
||||
impala-shell -q "create database if not exists ${SHADOW}"
|
||||
impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
|
||||
impala-shell -d ${TARGET} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${TARGET}.\1;/" | impala-shell -f -
|
||||
echo "Shadow db ready!"
|
||||
|
|
@ -45,35 +45,3 @@ FROM ${stats_db_name}.dataset
|
|||
UNION ALL
|
||||
SELECT *, bestlicence AS access_mode
|
||||
FROM ${stats_db_name}.otherresearchproduct;
|
||||
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
-- To see with Antonis if the following is needed and where it should be placed
|
||||
-------------------------------------------------------------------------------
|
||||
CREATE TABLE ${stats_db_name}.numbers_country AS
|
||||
SELECT org.country AS country, count(distinct rd.datasource) AS datasources, count(distinct r.id) AS publications
|
||||
FROM ${stats_db_name}.result r,
|
||||
${stats_db_name}.result_datasources rd,
|
||||
${stats_db_name}.datasource d,
|
||||
${stats_db_name}.datasource_organizations dor,
|
||||
${stats_db_name}.organization org
|
||||
WHERE r.id = rd.id
|
||||
AND rd.datasource = d.id
|
||||
AND d.id = dor.id
|
||||
AND dor.organization = org.id
|
||||
AND r.type = 'publication'
|
||||
AND r.bestlicence = 'Open Access'
|
||||
GROUP BY org.country;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -59,33 +59,4 @@ from result_gold
|
|||
union all
|
||||
select distinct r.id, false as gold
|
||||
from ${stats_db_name}.result r
|
||||
where r.id not in (select id from result_gold);
|
||||
|
||||
-- shortcut result-country through the organization affiliation
|
||||
create table ${stats_db_name}.result_affiliated_country as
|
||||
select r.id as id, o.country as country
|
||||
from ${stats_db_name}.result r
|
||||
join ${stats_db_name}.result_organization ro on ro.id=r.id
|
||||
join ${stats_db_name}.organization o on o.id=ro.organization
|
||||
where o.country is not null and o.country!='';
|
||||
|
||||
-- shortcut result-country through datasource of deposition
|
||||
create table ${stats_db_name}.result_deposited_country as
|
||||
select r.id as id, o.country as country
|
||||
from ${stats_db_name}.result r
|
||||
join ${stats_db_name}.result_datasources rd on rd.id=r.id
|
||||
join ${stats_db_name}.datasource d on d.id=rd.datasource
|
||||
join ${stats_db_name}.datasource_organizations dor on dor.id=d.id
|
||||
join ${stats_db_name}.organization o on o.id=dor.organization
|
||||
where o.country is not null and o.country!='';
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
where r.id not in (select id from result_gold);
|
||||
|
|
@ -52,7 +52,4 @@ LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
|
|||
drop table if exists ${stats_db_name}.result;
|
||||
drop view if exists ${stats_db_name}.result;
|
||||
create table ${stats_db_name}.result stored as parquet as select * from ${stats_db_name}.result_tmp;
|
||||
drop table ${stats_db_name}.result_tmp;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS FOR COLUMNS;
|
||||
drop table ${stats_db_name}.result_tmp;
|
||||
|
|
@ -19,9 +19,6 @@ create table TARGET.result as
|
|||
select * from SOURCE.result r where exists (select 1 from SOURCE.result_concepts rc where rc.id=r.id) ) foo;
|
||||
compute stats TARGET.result;
|
||||
|
||||
create table TARGET.result_affiliated_country as select * from SOURCE.result_affiliated_country rac where exists (select 1 from TARGET.result r where r.id=rac.id);
|
||||
compute stats TARGET.result_affiliated_country;
|
||||
|
||||
create table TARGET.result_citations as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id);
|
||||
compute stats TARGET.result_citations;
|
||||
|
||||
|
|
@ -34,9 +31,6 @@ compute stats TARGET.result_concepts;
|
|||
create table TARGET.result_datasources as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id);
|
||||
compute stats TARGET.result_datasources;
|
||||
|
||||
create table TARGET.result_deposited_country as select * from SOURCE.result_deposited_country orig where exists (select 1 from TARGET.result r where r.id=orig.id);
|
||||
compute stats TARGET.result_deposited_country;
|
||||
|
||||
create table TARGET.result_fundercount as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id);
|
||||
compute stats TARGET.result_fundercount;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,259 @@
|
|||
create table TARGET.result_affiliated_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, c.code, c.name;
|
||||
|
||||
create table TARGET.result_affiliated_year stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, r.year
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year;
|
||||
|
||||
create table TARGET.result_affiliated_year_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, r.year, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year, c.code, c.name;
|
||||
|
||||
create table TARGET.result_affiliated_datasource stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, d.name as dname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_datasources rd on rd.id=r.id
|
||||
left outer join SOURCE.datasource d on d.id=rd.datasource
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name;
|
||||
|
||||
create table TARGET.result_affiliated_datasource_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, d.name as dname, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_datasources rd on rd.id=r.id
|
||||
left outer join SOURCE.datasource d on d.id=rd.datasource
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name, c.code, c.name;
|
||||
|
||||
create table TARGET.result_affiliated_organization stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, o.name as oname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name;
|
||||
|
||||
create table TARGET.result_affiliated_organization_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, o.name as oname, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name, c.code, c.name;
|
||||
|
||||
create table TARGET.result_affiliated_funder stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, p.funder as pfunder
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
join SOURCE.result_projects rp on rp.id=r.id
|
||||
join SOURCE.project p on p.id=rp.project
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder;
|
||||
|
||||
create table TARGET.result_affiliated_funder_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, p.funder as pfunder, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_organization ro on ro.id=r.id
|
||||
join SOURCE.organization o on o.id=ro.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
join SOURCE.result_projects rp on rp.id=r.id
|
||||
join SOURCE.project p on p.id=rp.project
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder, c.code, c.name;
|
||||
|
||||
create table TARGET.result_deposited_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, c.code, c.name;
|
||||
|
||||
create table TARGET.result_deposited_year stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, r.year
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year;
|
||||
|
||||
create table TARGET.result_deposited_year_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, r.year, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year, c.code, c.name;
|
||||
|
||||
create table TARGET.result_deposited_datasource stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, d.name as dname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name;
|
||||
|
||||
create table TARGET.result_deposited_datasource_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, d.name as dname, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name, c.code, c.name;
|
||||
|
||||
create table TARGET.result_deposited_organization stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, o.name as oname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name;
|
||||
|
||||
create table TARGET.result_deposited_organization_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, o.name as oname, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name, c.code, c.name;
|
||||
|
||||
create table TARGET.result_deposited_funder stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, p.funder as pfunder
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
join SOURCE.result_projects rp on rp.id=r.id
|
||||
join SOURCE.project p on p.id=rp.project
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder;
|
||||
|
||||
create table TARGET.result_deposited_funder_country stored as parquet as
|
||||
select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
|
||||
case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
|
||||
r.peer_reviewed, r.type, p.funder as pfunder, c.code as ccode, c.name as cname
|
||||
from SOURCE.result r
|
||||
join SOURCE.result_datasources rd on rd.id=r.id
|
||||
join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
|
||||
join SOURCE.datasource_organizations dor on dor.id=d.id
|
||||
join SOURCE.organization o on o.id=dor.organization
|
||||
join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
|
||||
join SOURCE.result_projects rp on rp.id=r.id
|
||||
join SOURCE.project p on p.id=rp.project
|
||||
left outer join SOURCE.result_licenses rl on rl.id=r.id
|
||||
left outer join SOURCE.result_pids pids on pids.id=r.id
|
||||
group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder, c.code, c.name;
|
||||
|
||||
compute stats TARGET.result_affiliated_country;
|
||||
compute stats TARGET.result_affiliated_year;
|
||||
compute stats TARGET.result_affiliated_year_country;
|
||||
compute stats TARGET.result_affiliated_datasource;
|
||||
compute stats TARGET.result_affiliated_datasource_country;
|
||||
compute stats TARGET.result_affiliated_organization;
|
||||
compute stats TARGET.result_affiliated_organization_country;
|
||||
compute stats TARGET.result_affiliated_funder;
|
||||
compute stats TARGET.result_affiliated_funder_country;
|
||||
compute stats TARGET.result_deposited_country;
|
||||
compute stats TARGET.result_deposited_year;
|
||||
compute stats TARGET.result_deposited_year_country;
|
||||
compute stats TARGET.result_deposited_datasource;
|
||||
compute stats TARGET.result_deposited_datasource_country;
|
||||
compute stats TARGET.result_deposited_organization;
|
||||
compute stats TARGET.result_deposited_organization_country;
|
||||
compute stats TARGET.result_deposited_funder;
|
||||
compute stats TARGET.result_deposited_funder_country;
|
||||
|
|
@ -25,6 +25,14 @@
|
|||
<name>monitor_db_shadow_name</name>
|
||||
<description>the name of the shadow monitor db</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_name</name>
|
||||
<description>the target monitor db name</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>observatory_db_shadow_name</name>
|
||||
<description>the name of the shadow monitor db</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>stats_tool_api_url</name>
|
||||
<description>The url of the API of the stats tool. Is used to trigger the cache update.</description>
|
||||
|
|
@ -305,11 +313,26 @@
|
|||
<argument>${wf:appPath()}/scripts/step20-createMonitorDB.sql</argument>
|
||||
<file>monitor.sh</file>
|
||||
</shell>
|
||||
<ok to="Step21"/>
|
||||
<ok to="step21-createObservatoryDB"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step21">
|
||||
<action name="step21-createObservatoryDB">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>observatory.sh</exec>
|
||||
<argument>${stats_db_name}</argument>
|
||||
<argument>${observatory_db_name}</argument>
|
||||
<argument>${observatory_db_shadow_name}</argument>
|
||||
<argument>${wf:appPath()}/scripts/step21-createObservatoryDB.sql</argument>
|
||||
<file>observatory.sh</file>
|
||||
</shell>
|
||||
<ok to="Step22"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step22">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
|
|
@ -322,4 +345,4 @@
|
|||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
</workflow-app>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project-shared-configuration>
|
||||
<!--
|
||||
This file contains additional configuration written by modules in the NetBeans IDE.
|
||||
The configuration is intended to be shared among all the users of project and
|
||||
therefore it is assumed to be part of version control checkout.
|
||||
Without this configuration present, some functionality in the IDE may be limited or fail altogether.
|
||||
-->
|
||||
<properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
|
||||
<!--
|
||||
Properties that influence various parts of the IDE, especially code formatting and the like.
|
||||
You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
|
||||
That way multiple projects can share the same settings (useful for formatting rules for example).
|
||||
Any value defined here will override the pom.xml file value but is only applicable to the current project.
|
||||
-->
|
||||
<netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
|
||||
</properties>
|
||||
</project-shared-configuration>
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
|
||||
<!-- <parent>
|
||||
<artifactId>dhp-workflows</artifactId >
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<version>1.1.7-SNAPSHOT</version>
|
||||
</parent>
|
||||
<groupId>eu.dnetlib</groupId> -->
|
||||
<!-- <parent>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<version>1.1.7-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dhp-usage-stats-update</artifactId> -->
|
||||
|
||||
<parent>
|
||||
<artifactId>dhp-workflows</artifactId>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<version>1.2.4-SNAPSHOT</version>
|
||||
<relativePath>../</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dhp-usage-datasets-stats-update</artifactId>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>pl.project13.maven</groupId>
|
||||
<artifactId>git-commit-id-plugin</artifactId>
|
||||
<version>2.1.15</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>revision</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
|
||||
<!-- more config here as you see fit -->
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.6.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<cdh.hive.version>0.13.1-cdh5.2.1</cdh.hive.version>
|
||||
<cdh.hadoop.version>2.5.0-cdh5.2.1</cdh.hadoop.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.11</artifactId>
|
||||
<version>2.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.11</artifactId>
|
||||
<version>2.4.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.googlecode.json-simple</groupId>
|
||||
<artifactId>json-simple</artifactId>
|
||||
<version>1.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20180130</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hive</groupId>
|
||||
<artifactId>hive-jdbc</artifactId>
|
||||
<version>${cdh.hive.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>2.7.4</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>eu.dnetlib.dhp</groupId>
|
||||
<artifactId>dhp-common</artifactId>
|
||||
<version>1.2.4-SNAPSHOT</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mchange</groupId>
|
||||
<artifactId>c3p0</artifactId>
|
||||
<version>0.9.5.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>c3p0</groupId>
|
||||
<artifactId>c3p0</artifactId>
|
||||
<version>0.9.1.2</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>1.7.26</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<name>dhp-usage-datasets-stats-update</name>
|
||||
</project>
|
||||
|
|
@ -0,0 +1 @@
|
|||
mvn clean package -Poozie-package,deploy,run -Dworkflow.source.dir=eu/dnetlib/dhp/oa/graph/datasetsusagestats
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
/**
|
||||
* @author D. Pierrakos
|
||||
*/
|
||||
/**
|
||||
* @author D. Pierrakos
|
||||
*/
|
||||
import com.mchange.v2.c3p0.ComboPooledDataSource;
|
||||
|
||||
public abstract class ConnectDB {
|
||||
|
||||
public static Connection DB_HIVE_CONNECTION;
|
||||
public static Connection DB_IMPALA_CONNECTION;
|
||||
|
||||
private static String dbHiveUrl;
|
||||
private static String dbImpalaUrl;
|
||||
private static String datasetUsageStatsDBSchema;
|
||||
private static String datasetsUsageStatsPermanentDBSchema;
|
||||
private static String statsDBSchema;
|
||||
private final static Logger logger = Logger.getLogger(ConnectDB.class);
|
||||
private Statement stmt = null;
|
||||
|
||||
static void init() throws ClassNotFoundException {
|
||||
|
||||
dbHiveUrl = ExecuteWorkflow.dbHiveUrl;
|
||||
dbImpalaUrl = ExecuteWorkflow.dbImpalaUrl;
|
||||
datasetUsageStatsDBSchema = ExecuteWorkflow.datasetUsageStatsDBSchema;
|
||||
datasetsUsageStatsPermanentDBSchema = ExecuteWorkflow.datasetsUsageStatsPermanentDBSchema;
|
||||
statsDBSchema = ExecuteWorkflow.statsDBSchema;
|
||||
|
||||
Class.forName("org.apache.hive.jdbc.HiveDriver");
|
||||
}
|
||||
|
||||
public static Connection getHiveConnection() throws SQLException {
|
||||
if (DB_HIVE_CONNECTION != null && !DB_HIVE_CONNECTION.isClosed()) {
|
||||
return DB_HIVE_CONNECTION;
|
||||
} else {
|
||||
DB_HIVE_CONNECTION = connectHive();
|
||||
|
||||
return DB_HIVE_CONNECTION;
|
||||
}
|
||||
}
|
||||
|
||||
public static Connection getImpalaConnection() throws SQLException {
|
||||
if (DB_IMPALA_CONNECTION != null && !DB_IMPALA_CONNECTION.isClosed()) {
|
||||
return DB_IMPALA_CONNECTION;
|
||||
} else {
|
||||
DB_IMPALA_CONNECTION = connectImpala();
|
||||
|
||||
return DB_IMPALA_CONNECTION;
|
||||
}
|
||||
}
|
||||
|
||||
public static String getDataSetUsageStatsDBSchema() {
|
||||
String datePattern = "YYYYMMdd";
|
||||
DateFormat df = new SimpleDateFormat(datePattern);
|
||||
// Get the today date using Calendar object.
|
||||
Date today = Calendar.getInstance().getTime();
|
||||
String todayAsString = df.format(today);
|
||||
|
||||
return ConnectDB.datasetUsageStatsDBSchema + "_" + todayAsString;
|
||||
}
|
||||
|
||||
public static String getStatsDBSchema() {
|
||||
return ConnectDB.statsDBSchema;
|
||||
}
|
||||
|
||||
public static String getDatasetsUsagestatsPermanentDBSchema() {
|
||||
return ConnectDB.datasetsUsageStatsPermanentDBSchema;
|
||||
}
|
||||
|
||||
private static Connection connectHive() throws SQLException {
|
||||
logger.info("trying to open Hive connection...");
|
||||
|
||||
ComboPooledDataSource cpds = new ComboPooledDataSource();
|
||||
cpds.setJdbcUrl(dbHiveUrl);
|
||||
cpds.setUser("dimitris.pierrakos");
|
||||
cpds.setAcquireIncrement(1);
|
||||
cpds.setMaxPoolSize(100);
|
||||
cpds.setMinPoolSize(1);
|
||||
cpds.setInitialPoolSize(1);
|
||||
cpds.setMaxIdleTime(300);
|
||||
cpds.setMaxConnectionAge(36000);
|
||||
|
||||
cpds.setAcquireRetryAttempts(5);
|
||||
cpds.setAcquireRetryDelay(2000);
|
||||
cpds.setBreakAfterAcquireFailure(false);
|
||||
|
||||
cpds.setCheckoutTimeout(0);
|
||||
cpds.setPreferredTestQuery("SELECT 1");
|
||||
cpds.setIdleConnectionTestPeriod(60);
|
||||
|
||||
logger.info("Opened HIVE successfully");
|
||||
|
||||
return cpds.getConnection();
|
||||
// Connection connection = DriverManager.getConnection(dbHiveUrl);
|
||||
// logger.debug("Opened Hive successfully");
|
||||
//
|
||||
// return connection;
|
||||
|
||||
}
|
||||
|
||||
private static Connection connectImpala() throws SQLException {
|
||||
logger.info("trying to open Impala connection...");
|
||||
ComboPooledDataSource cpds = new ComboPooledDataSource();
|
||||
cpds.setJdbcUrl(dbImpalaUrl);
|
||||
cpds.setUser("dimitris.pierrakos");
|
||||
cpds.setAcquireIncrement(1);
|
||||
cpds.setMaxPoolSize(100);
|
||||
cpds.setMinPoolSize(1);
|
||||
cpds.setInitialPoolSize(1);
|
||||
cpds.setMaxIdleTime(300);
|
||||
cpds.setMaxConnectionAge(36000);
|
||||
|
||||
cpds.setAcquireRetryAttempts(5);
|
||||
cpds.setAcquireRetryDelay(2000);
|
||||
cpds.setBreakAfterAcquireFailure(false);
|
||||
|
||||
cpds.setCheckoutTimeout(0);
|
||||
cpds.setPreferredTestQuery("SELECT 1");
|
||||
cpds.setIdleConnectionTestPeriod(60);
|
||||
|
||||
logger.info("Opened Impala successfully");
|
||||
return cpds.getConnection();
|
||||
// Connection connection = DriverManager.getConnection(dbHiveUrl);
|
||||
// logger.debug("Opened Impala successfully");
|
||||
//
|
||||
// return connection;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import java.sql.Statement;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author D. Pierrakos
|
||||
*/
|
||||
public class DatasetsStatsDB {
|
||||
|
||||
private String logPath;
|
||||
private String logRepoPath;
|
||||
private String logPortalPath;
|
||||
|
||||
private Statement stmt = null;
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DatasetsStatsDB.class);
|
||||
|
||||
public DatasetsStatsDB(String logRepoPath, String logPortalPath) throws Exception {
|
||||
this.logRepoPath = logRepoPath;
|
||||
this.logPortalPath = logPortalPath;
|
||||
|
||||
}
|
||||
|
||||
public void recreateDBAndTables() throws Exception {
|
||||
this.createDatabase();
|
||||
this.createTables();
|
||||
}
|
||||
|
||||
private void createDatabase() throws Exception {
|
||||
try {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger.info("Dropping datasets DB: " + ConnectDB.getDataSetUsageStatsDBSchema());
|
||||
String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema() + " CASCADE";
|
||||
stmt.executeUpdate(dropDatabase);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to drop database: " + e);
|
||||
throw new Exception("Failed to drop database: " + e.toString(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger.info("Creating datacite usagestats DB: " + ConnectDB.getDataSetUsageStatsDBSchema());
|
||||
String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema();
|
||||
stmt.executeUpdate(createDatabase);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to create database: " + e);
|
||||
throw new Exception("Failed to create database: " + e.toString(), e);
|
||||
}
|
||||
try {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger
|
||||
.info(
|
||||
"Creating permanent datasets usagestats DB: " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema());
|
||||
String createPermanentDatabase = "CREATE DATABASE IF NOT EXISTS "
|
||||
+ ConnectDB.getDatasetsUsagestatsPermanentDBSchema();
|
||||
stmt.executeUpdate(createPermanentDatabase);
|
||||
logger
|
||||
.info(
|
||||
"Created permanent datasets usagestats DB: " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema());
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to create database: " + e);
|
||||
throw new Exception("Failed to create database: " + e.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void createTables() throws Exception {
|
||||
try {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
// Create Reports table - This table should exist
|
||||
logger.info("Creating Reports Tmp Table");
|
||||
String sqlCreateTableDataciteReports = "CREATE TABLE IF NOT EXISTS "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacitereports_tmp(reportid STRING, \n"
|
||||
+ " name STRING, \n"
|
||||
+ " source STRING,\n"
|
||||
+ " release STRING,\n"
|
||||
+ " createdby STRING,\n"
|
||||
+ " report_start_date STRING,\n"
|
||||
+ " report_end_date STRING)\n"
|
||||
+ " CLUSTERED BY (reportid)\n"
|
||||
+ " into 100 buckets stored as orc tblproperties('transactional'='true')";
|
||||
|
||||
stmt.executeUpdate(sqlCreateTableDataciteReports);
|
||||
logger.info("Reports Table Created");
|
||||
|
||||
// Create Datasets Performance Table
|
||||
logger.info("Creating DataSetsPerformance Tmp Table");
|
||||
String sqlCreateTableDataSetsPerformance = "CREATE TABLE IF NOT EXISTS "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datasetsperformance_tmp(ds_type STRING,\n"
|
||||
+ " ds_title STRING,\n"
|
||||
+ " yop STRING,\n"
|
||||
+ " dataset_type STRING, \n"
|
||||
+ " uri STRING,\n"
|
||||
+ " platform STRING,\n"
|
||||
+ " publisher STRING,\n"
|
||||
+ " publisher_id array<struct<type:STRING, value:STRING>>,\n"
|
||||
+ " dataset_contributors array<struct<type:STRING, value:STRING>>,\n"
|
||||
+ " period_end STRING,\n"
|
||||
+ " period_from STRING,\n"
|
||||
+ " access_method STRING,\n"
|
||||
+ " metric_type STRING,\n"
|
||||
+ " count INT,\n"
|
||||
+ " reportid STRING)\n"
|
||||
+ " CLUSTERED BY (ds_type)\n"
|
||||
+ " into 100 buckets stored as orc tblproperties('transactional'='true')";
|
||||
stmt.executeUpdate(sqlCreateTableDataSetsPerformance);
|
||||
logger.info("DataSetsPerformance Tmp Table Created");
|
||||
|
||||
logger.info("Creating Datacite Reports table");
|
||||
String createDataciteReportsTable = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacitereports LIKE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacitereports_tmp STORED AS PARQUET";
|
||||
stmt.executeUpdate(createDataciteReportsTable);
|
||||
logger.info("Datacite Reports Table created");
|
||||
|
||||
logger.info("Creating Datasets Performance table");
|
||||
String createDatasetPerformanceTable = "CREATE TABLE IF NOT EXISTS "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datasetsperformance LIKE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datasetsperformance_tmp STORED AS PARQUET";
|
||||
stmt.executeUpdate(createDatasetPerformanceTable);
|
||||
logger.info("DatasetsPerformance Table created");
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getHiveConnection().close();
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to create tables: " + e);
|
||||
throw new Exception("Failed to create tables: " + e.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.json.simple.parser.ParseException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
|
||||
/**
|
||||
* @author D.Pierrakos
|
||||
*/
|
||||
public class DownloadReportsListFromDatacite {
|
||||
|
||||
private String dataciteBaseURL;
|
||||
private String dataciteReportPath;
|
||||
private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
|
||||
|
||||
public DownloadReportsListFromDatacite(String dataciteBaseURL, String dataciteReportPath)
|
||||
throws MalformedURLException, Exception {
|
||||
|
||||
this.dataciteBaseURL = dataciteBaseURL;
|
||||
this.dataciteReportPath = dataciteReportPath;
|
||||
}
|
||||
|
||||
public void downloadReportsList() throws ParseException {
|
||||
StringBuilder responseStrBuilder = new StringBuilder();
|
||||
|
||||
Gson gson = new Gson();
|
||||
|
||||
try {
|
||||
BufferedInputStream in = new BufferedInputStream(new URL(dataciteBaseURL).openStream());
|
||||
BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
|
||||
String inputStr;
|
||||
|
||||
while ((inputStr = streamReader.readLine()) != null) {
|
||||
responseStrBuilder.append(inputStr);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.info(e.getMessage());
|
||||
}
|
||||
JsonObject jsonObject = gson.fromJson(responseStrBuilder.toString(), JsonObject.class);
|
||||
JsonArray dataArray = jsonObject.getAsJsonArray("reports");
|
||||
ArrayList reportsList = new ArrayList();
|
||||
for (JsonElement element : dataArray) {
|
||||
reportsList.add(element.getAsJsonObject().get("id").getAsString());
|
||||
}
|
||||
|
||||
Iterator it = reportsList.iterator();
|
||||
while (it.hasNext()) {
|
||||
String reportId = it.next().toString();
|
||||
String url = dataciteBaseURL + reportId;
|
||||
|
||||
try {
|
||||
BufferedInputStream in = new BufferedInputStream(new URL(url).openStream());
|
||||
BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
|
||||
String inputStr;
|
||||
StringBuilder responseStrBuilder2 = new StringBuilder();
|
||||
while ((inputStr = streamReader.readLine()) != null) {
|
||||
responseStrBuilder2.append(inputStr);
|
||||
}
|
||||
FileSystem fs = FileSystem.get(new Configuration());
|
||||
FSDataOutputStream fin = fs
|
||||
.create(
|
||||
new Path(dataciteReportPath + "/" + reportId + ".json"),
|
||||
true);
|
||||
byte[] jsonObjectRawBytes = responseStrBuilder2.toString().getBytes();
|
||||
fin.write(jsonObjectRawBytes);
|
||||
fin.writeChar('\n');
|
||||
|
||||
fin.close();
|
||||
|
||||
fin.close();
|
||||
} catch (IOException e) {
|
||||
System.out.println(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.log4j.BasicConfigurator;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
|
||||
/**
|
||||
* @author D. Pierrakos, S. Zoupanos
|
||||
*/
|
||||
public class ExecuteWorkflow {
|
||||
|
||||
static String dataciteBaseURL;
|
||||
static String dataciteReportPath;
|
||||
static String dbHiveUrl;
|
||||
static String dbImpalaUrl;
|
||||
static String datasetUsageStatsDBSchema;
|
||||
static String datasetsUsageStatsPermanentDBSchema;
|
||||
static String statsDBSchema;
|
||||
static boolean recreateDbAndTables;
|
||||
static boolean datasetsEmptyDirs;
|
||||
static boolean finalTablesVisibleToImpala;
|
||||
|
||||
public static void main(String args[]) throws Exception {
|
||||
|
||||
// Sending the logs to the console
|
||||
BasicConfigurator.configure();
|
||||
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
UsageStatsExporter.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/oa/graph/datasetsusagestats/export/datasets_usagestats_parameters.json")));
|
||||
parser.parseArgument(args);
|
||||
|
||||
// Setting up the initial parameters
|
||||
dataciteBaseURL = parser.get("dataciteBaseURL");
|
||||
dataciteReportPath = parser.get("dataciteReportPath");
|
||||
dbHiveUrl = parser.get("dbHiveUrl");
|
||||
dbImpalaUrl = parser.get("dbImpalaUrl");
|
||||
datasetUsageStatsDBSchema = parser.get("datasetUsageStatsDBSchema");
|
||||
datasetsUsageStatsPermanentDBSchema = parser.get("datasetsUsageStatsPermanentDBSchema");
|
||||
statsDBSchema = parser.get("statsDBSchema");
|
||||
|
||||
if (parser.get("recreateDbAndTables").toLowerCase().equals("true"))
|
||||
recreateDbAndTables = true;
|
||||
else
|
||||
recreateDbAndTables = false;
|
||||
|
||||
if (parser.get("datasetsEmptyDirs").toLowerCase().equals("true"))
|
||||
datasetsEmptyDirs = true;
|
||||
else
|
||||
datasetsEmptyDirs = false;
|
||||
|
||||
if (parser.get("finalTablesVisibleToImpala").toLowerCase().equals("true"))
|
||||
finalTablesVisibleToImpala = true;
|
||||
else
|
||||
finalTablesVisibleToImpala = false;
|
||||
|
||||
UsageStatsExporter usagestatsExport = new UsageStatsExporter();
|
||||
usagestatsExport.export();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,388 @@
|
|||
/*
|
||||
* To change this license header, choose License Headers in Project Properties.
|
||||
* To change this template file, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.MalformedURLException;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* @author D.Pierrakos
|
||||
*/
|
||||
public class ReadReportsListFromDatacite {
|
||||
|
||||
private String dataciteReportPath;
|
||||
private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
|
||||
|
||||
public ReadReportsListFromDatacite(String dataciteReportPath) throws MalformedURLException, Exception {
|
||||
|
||||
this.dataciteReportPath = dataciteReportPath;
|
||||
}
|
||||
|
||||
public void readReports() throws Exception {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
ArrayList<String> jsonFiles = listHdfsDir(dataciteReportPath);
|
||||
for (String jsonFile : jsonFiles) {
|
||||
logger.info("Reading report file " + jsonFile);
|
||||
this.createTmpReportsTable(jsonFile);
|
||||
|
||||
String sqlSelectReportID = "SELECT get_json_object(json, '$.report.id') FROM "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(sqlSelectReportID);
|
||||
ResultSet rstmpReportID = stmt.getResultSet();
|
||||
|
||||
String reportID = null;
|
||||
while (rstmpReportID.next()) {
|
||||
reportID = rstmpReportID.getString(1);
|
||||
}
|
||||
|
||||
logger.info("Checking report with id " + reportID);
|
||||
String sqlCheckIfReportExists = "SELECT source FROM " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacitereports_tmp where reportid=?";
|
||||
PreparedStatement stGetReportID = ConnectDB.getHiveConnection().prepareStatement(sqlCheckIfReportExists);
|
||||
stGetReportID.setString(1, reportID);
|
||||
|
||||
ResultSet rsCheckIfReportExist = stGetReportID.executeQuery();
|
||||
|
||||
if (rsCheckIfReportExist.next()) {
|
||||
logger.info("Report found with ID " + reportID);
|
||||
dropTmpReportsTable();
|
||||
} else {
|
||||
String sqlInsertReport = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ " .datacitereports_tmp "
|
||||
+ "SELECT\n"
|
||||
+ " get_json_object(json, '$.report.id') AS reportid,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.report-name') AS name,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.report-id') AS source,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.release') AS release,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.created-by\') AS createdby,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.reporting-period.begin-date') AS fromdate,\n"
|
||||
+ " get_json_object(json, '$.report.report-header.reporting-period.end-date') AS todate \n"
|
||||
+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(sqlInsertReport);
|
||||
|
||||
logger.info("Report added");
|
||||
|
||||
logger.info("Adding datasets");
|
||||
String sqlSelecteDatasetsArray = "SELECT get_json_object(json, '$.report.report-datasets') FROM "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(sqlSelecteDatasetsArray);
|
||||
ResultSet rstmpReportDatasets = stmt.getResultSet();
|
||||
|
||||
if (rstmpReportDatasets.next() && rstmpReportDatasets.getString(1).indexOf(',') > 0) {
|
||||
// String[] listDatasets = rstmpReportDatasets.getString(1).split(",");
|
||||
// String listDatasets = rstmpReportDatasets.getString(1);
|
||||
String sqlSelectReport = "SELECT * FROM "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(sqlSelectReport);
|
||||
ResultSet rstmpReportAll = stmt.getResultSet();
|
||||
if (rstmpReportAll.next()) {
|
||||
String listDatasets = rstmpReportAll.getString(1);
|
||||
logger.info("Adding uncompressed performance for " + reportID);
|
||||
this.readDatasetsReport(listDatasets, reportID);
|
||||
}
|
||||
|
||||
}
|
||||
logger.info("Adding gziped performance for datasets");
|
||||
String sqlSelecteReportSubsets = "SELECT get_json_object(json, '$.report.report-subsets.gzip[0]') FROM "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(sqlSelecteReportSubsets);
|
||||
ResultSet rstmpReportSubsets = stmt.getResultSet();
|
||||
if (rstmpReportSubsets.next()) {
|
||||
String unCompressedReport = uncompressString(rstmpReportSubsets.getString(1));
|
||||
this.readDatasetsReport(unCompressedReport, reportID);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.dropTmpReportsTable();
|
||||
}
|
||||
|
||||
public void readDatasetsReport(String prettyDatasetsReports, String reportId) throws Exception {
|
||||
logger.info("Reading Datasets performance for report " + reportId);
|
||||
logger.info("Write Performance Report To File");
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
JsonNode jsonNode = objectMapper.readValue(prettyDatasetsReports, JsonNode.class);
|
||||
String datasetsReports = jsonNode.toString();
|
||||
String report = datasetsReports
|
||||
.replace("report-datasets", "report_datasets")
|
||||
.replace("dataset-title", "dataset_title")
|
||||
.replace("dataset-id", "dataset_id")
|
||||
.replace("data-type", "data_type")
|
||||
.replace("publisher-id", "publisher_id")
|
||||
.replace("dataset-contributors", "dataset_contributors")
|
||||
.replace("begin-date", "begin_date")
|
||||
.replace("end-date", "end_date")
|
||||
.replace("access-method", "access_method")
|
||||
.replace("metric-type", "metric_type")
|
||||
.replace("doi:", "");
|
||||
FileSystem fs = FileSystem.get(new Configuration());
|
||||
String tmpPath = dataciteReportPath + "/tmpjson";
|
||||
FSDataOutputStream fin = fs
|
||||
.create(new Path(dataciteReportPath + "/tmpjson/" + reportId + "_Compressed.json"), true);
|
||||
byte[] jsonObjectRawBytes = report.getBytes();
|
||||
|
||||
fin.write(jsonObjectRawBytes);
|
||||
|
||||
fin.writeChar('\n');
|
||||
fin.close();
|
||||
|
||||
logger.info("Reading Performance Report From File...");
|
||||
|
||||
String sqlCreateTempTableForDatasets = "CREATE TEMPORARY TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".tmpjsoncompressesed (report_datasets array<struct<dataset_id:array<struct<value:string>>,dataset_title:string, data_type:string, "
|
||||
+ "uri:string, publisher:string, publisher_id:array<struct<type:string, value:string>>,platform:string, yop:string, "
|
||||
+ "dataset_contributors:array<struct<type:string, value:string>>,"
|
||||
+ "performance:array<struct<period:struct<begin_date:string,end_date:string>, "
|
||||
+ "instance:array<struct<count:int,access_method:string,metric_type:string>>>>>>) "
|
||||
+ "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n"
|
||||
+ "LOCATION '" + tmpPath + "'";
|
||||
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Adding JSON Serde jar");
|
||||
stmt.executeUpdate("add jar /usr/share/cmf/common_jars/hive-hcatalog-core-1.1.0-cdh5.14.0.jar");
|
||||
logger.info("Added JSON Serde jar");
|
||||
|
||||
logger.info("Inserting Datasets Performance");
|
||||
stmt.execute(sqlCreateTempTableForDatasets);
|
||||
|
||||
String sqlInsertToDatasetsPerformance = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datasetsperformance_tmp SELECT dataset.dataset_id[0].value ds_type, "
|
||||
+ " dataset.dataset_title ds_title, "
|
||||
+ " dataset.yop yop, "
|
||||
+ " dataset.data_type dataset_type, "
|
||||
+ " dataset.uri uri, "
|
||||
+ " dataset.platform platform, "
|
||||
+ " dataset.publisher publisher, "
|
||||
+ " dataset.publisher_id publisher_id, "
|
||||
+ " dataset.dataset_contributors dataset_contributors, "
|
||||
+ " period.end_date period_end, "
|
||||
+ " period.begin_date period_from, "
|
||||
+ " performance.access_method access_method, "
|
||||
+ " performance.metric_type metric_type, "
|
||||
+ " performance.count count, "
|
||||
+ "'" + reportId + "' report_id "
|
||||
+ " FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsoncompressesed "
|
||||
+ " LATERAL VIEW explode(report_datasets) exploded_table as dataset LATERAL VIEW explode(dataset.performance[0].instance) exploded_table2 as performance "
|
||||
+ " LATERAL VIEW explode (array(dataset.performance[0].period)) exploded_table3 as period";
|
||||
|
||||
stmt.executeUpdate(sqlInsertToDatasetsPerformance);
|
||||
|
||||
logger.info("Datasets Performance Inserted for Report " + reportId);
|
||||
|
||||
stmt.execute("Drop table " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsoncompressesed");
|
||||
|
||||
logger.info("Datasets Report Added");
|
||||
|
||||
}
|
||||
|
||||
private ArrayList<String> listHdfsDir(String dir) throws Exception {
|
||||
|
||||
FileSystem hdfs = FileSystem.get(new Configuration());
|
||||
RemoteIterator<LocatedFileStatus> Files;
|
||||
ArrayList<String> fileNames = new ArrayList<>();
|
||||
|
||||
try {
|
||||
Path exportPath = new Path(hdfs.getUri() + dir);
|
||||
Files = hdfs.listFiles(exportPath, false);
|
||||
while (Files.hasNext()) {
|
||||
String fileName = Files.next().getPath().toString();
|
||||
fileNames.add(fileName);
|
||||
}
|
||||
|
||||
hdfs.close();
|
||||
} catch (Exception e) {
|
||||
logger.error("HDFS file path with exported data does not exist : " + new Path(hdfs.getUri() + dir));
|
||||
throw new Exception("HDFS file path with exported data does not exist : " + dir, e);
|
||||
}
|
||||
|
||||
return fileNames;
|
||||
}
|
||||
|
||||
private String readHDFSFile(String filename) throws Exception {
|
||||
String result;
|
||||
try {
|
||||
|
||||
FileSystem fs = FileSystem.get(new Configuration());
|
||||
// log.info("reading file : " + filename);
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(filename))));
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String line = br.readLine();
|
||||
|
||||
while (line != null) {
|
||||
sb.append(line);
|
||||
// sb.append(line);
|
||||
line = br.readLine();
|
||||
}
|
||||
// uncompressedReport = sb.toString().replace("][{\"idSite\"", ",{\"idSite\"");
|
||||
result = sb.toString().trim();
|
||||
// fs.close();
|
||||
} catch (Exception e) {
|
||||
throw new Exception(e);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static String uncompressString(String zippedBase64Str)
|
||||
throws IOException {
|
||||
String uncompressedReport = null;
|
||||
|
||||
byte[] bytes = Base64.getDecoder().decode(zippedBase64Str);
|
||||
GZIPInputStream zi = null;
|
||||
try {
|
||||
zi = new GZIPInputStream(new ByteArrayInputStream(bytes));
|
||||
uncompressedReport = IOUtils.toString(zi);
|
||||
} finally {
|
||||
IOUtils.closeQuietly(zi);
|
||||
}
|
||||
logger.info("Report Succesfully Uncompressed...");
|
||||
return uncompressedReport;
|
||||
}
|
||||
|
||||
private void createTmpReportsTable(String jsonFile) throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
dropTmpReportsTable();
|
||||
String createTmpTable = "CREATE TEMPORARY TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".tmpjsonToTable (json STRING)";
|
||||
stmt.executeUpdate(createTmpTable);
|
||||
logger.info("Temporary Table for Json Report Created");
|
||||
|
||||
String insertJsonReport = "LOAD DATA INPATH '" + jsonFile + "' INTO TABLE "
|
||||
+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
stmt.execute(insertJsonReport);
|
||||
logger.info("JSON Report File inserted to tmpjsonToTable Table");
|
||||
}
|
||||
|
||||
private void dropTmpReportsTable() throws SQLException {
|
||||
logger.info("Dropping tmpjson Table");
|
||||
String dropTmpTable = "DROP TABLE IF EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
stmt.executeUpdate(dropTmpTable);
|
||||
logger.info("Dropped Table for Json Report Table");
|
||||
|
||||
}
|
||||
|
||||
public void createUsageStatisticsTable() throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger.info("Updating Datacite Reports table");
|
||||
String createDataciteReportsTable = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacitereports "
|
||||
+ "SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports_tmp";
|
||||
stmt.executeUpdate(createDataciteReportsTable);
|
||||
logger.info("Datacite Reports Table updated");
|
||||
|
||||
logger.info("Updating Datasets Performance table");
|
||||
String createDatasetPerformanceTable = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datasetsperformance "
|
||||
+ "SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance_tmp";
|
||||
stmt.executeUpdate(createDatasetPerformanceTable);
|
||||
logger.info("DatasetsPerformance Table updated");
|
||||
|
||||
logger.info("Creating Downloads Stats table");
|
||||
String createDownloadsTable = "CREATE TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacite_downloads STORED AS PARQUET as "
|
||||
+ "SELECT 'Datacite' source, d.id repository_id, od.id result_id, regexp_replace(substring(string(period_end),0,7),'-','/') date, count, '0' openaire "
|
||||
+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance "
|
||||
+ "JOIN " + ConnectDB.getStatsDBSchema() + ".datasource d on name=platform "
|
||||
+ "JOIN " + ConnectDB.getStatsDBSchema() + ".result_oids od on string(ds_type)=od.oid "
|
||||
+ "where metric_type='total-dataset-requests' ";
|
||||
stmt.executeUpdate(createDownloadsTable);
|
||||
logger.info("Downloads Stats table created");
|
||||
|
||||
logger.info("Creating Views Stats table");
|
||||
String createViewsTable = "CREATE TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
|
||||
+ ".datacite_views STORED AS PARQUET as "
|
||||
+ "SELECT 'Datacite' source, d.id repository_id, od.id result_id, regexp_replace(substring(string(period_end),0,7),'-','/') date, count, '0' openaire "
|
||||
+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance "
|
||||
+ "JOIN " + ConnectDB.getStatsDBSchema() + ".datasource d on name=platform "
|
||||
+ "JOIN " + ConnectDB.getStatsDBSchema() + ".result_oids od on string(ds_type)=od.oid "
|
||||
+ "where metric_type='total-dataset-investigations' ";
|
||||
stmt.executeUpdate(createViewsTable);
|
||||
logger.info("Views Stats table created");
|
||||
|
||||
logger.info("Building Permanent Datasets Usage Stats DB");
|
||||
|
||||
logger.info("Dropping view datacitereports on permanent datacite usagestats DB");
|
||||
String sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped view datacitereports on permanent datacite usagestats DB");
|
||||
|
||||
logger.info("Create view datacitereports on permanent datacite usagestats DB");
|
||||
sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports"
|
||||
+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created view datacitereports on permanent datasets usagestats DB");
|
||||
|
||||
logger.info("Dropping view datasetsperformance on permanent datacite usagestats DB");
|
||||
sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped view datasetsperformance on permanent datacite usagestats DB");
|
||||
|
||||
logger.info("Create view datasetsperformance on permanent datacite usagestats DB");
|
||||
sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance"
|
||||
+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created view datasetsperformance on permanent datasets usagestats DB");
|
||||
|
||||
logger.info("Dropping view datacite_views on permanent datacite usagestats DB");
|
||||
sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped view datacite_views on permanent datacite usagestats DB");
|
||||
|
||||
logger.info("Create view datacite_views on permanent datacite usagestats DB");
|
||||
sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views"
|
||||
+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created view datacite_views on permanent datasets usagestats DB");
|
||||
|
||||
logger.info("Dropping view datacite_downloads on permanent datacite usagestats DB");
|
||||
sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped view datacite_downloads on permanent datacite usagestats DB");
|
||||
|
||||
logger.info("Create view datacite_downloads on permanent datacite usagestats DB");
|
||||
sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads"
|
||||
+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created view datacite_downloads on permanent datasets usagestats DB");
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getHiveConnection().close();
|
||||
logger.info("Completed Building Permanent Datasets Usage Stats DB");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
|
||||
package eu.dnetlib.oa.graph.datasetsusagestats.export;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Main class for downloading and processing Usage statistics
|
||||
*
|
||||
* @author D. Pierrakos, S. Zoupanos
|
||||
*/
|
||||
public class UsageStatsExporter {
|
||||
|
||||
private Statement stmt = null;
|
||||
|
||||
public UsageStatsExporter() {
|
||||
|
||||
}
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
|
||||
|
||||
private void reCreateLogDirs() throws IllegalArgumentException, IOException {
|
||||
FileSystem dfs = FileSystem.get(new Configuration());
|
||||
|
||||
logger.info("Deleting Log directory: " + ExecuteWorkflow.dataciteReportPath);
|
||||
dfs.delete(new Path(ExecuteWorkflow.dataciteReportPath), true);
|
||||
|
||||
logger.info("Creating Log directory: " + ExecuteWorkflow.dataciteReportPath);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.dataciteReportPath));
|
||||
|
||||
logger.info("Creating tmp directory: " + ExecuteWorkflow.dataciteReportPath + " " + "/tmpjson/");
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.dataciteReportPath + "/tmpjson/"));
|
||||
|
||||
}
|
||||
|
||||
public void export() throws Exception {
|
||||
|
||||
logger.info("Initialising DB properties");
|
||||
ConnectDB.init();
|
||||
ConnectDB.getHiveConnection();
|
||||
|
||||
if (ExecuteWorkflow.recreateDbAndTables) {
|
||||
DatasetsStatsDB datasetsDB = new DatasetsStatsDB("", "");
|
||||
datasetsDB.recreateDBAndTables();
|
||||
}
|
||||
logger.info("Initializing the download logs module");
|
||||
DownloadReportsListFromDatacite downloadReportsListFromDatacite = new DownloadReportsListFromDatacite(
|
||||
ExecuteWorkflow.dataciteBaseURL,
|
||||
ExecuteWorkflow.dataciteReportPath);
|
||||
|
||||
if (ExecuteWorkflow.datasetsEmptyDirs) {
|
||||
logger.info("Downloading Reports List From Datacite");
|
||||
this.reCreateLogDirs();
|
||||
downloadReportsListFromDatacite.downloadReportsList();
|
||||
logger.info("Reports List has been downloaded");
|
||||
}
|
||||
|
||||
ReadReportsListFromDatacite readReportsListFromDatacite = new ReadReportsListFromDatacite(
|
||||
ExecuteWorkflow.dataciteReportPath);
|
||||
logger.info("Store Reports To DB");
|
||||
readReportsListFromDatacite.readReports();
|
||||
logger.info("Reports Stored To DB");
|
||||
readReportsListFromDatacite.createUsageStatisticsTable();
|
||||
|
||||
// Make the tables available to Impala
|
||||
if (ExecuteWorkflow.finalTablesVisibleToImpala) {
|
||||
logger.info("Making tables visible to Impala");
|
||||
invalidateMetadata();
|
||||
}
|
||||
|
||||
logger.info("End");
|
||||
}
|
||||
|
||||
private void invalidateMetadata() throws SQLException {
|
||||
Statement stmt = null;
|
||||
|
||||
stmt = ConnectDB.getImpalaConnection().createStatement();
|
||||
|
||||
String sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
stmt.close();
|
||||
try {
|
||||
ConnectDB.getHiveConnection().close();
|
||||
} catch (Exception e) {
|
||||
logger.info("Message at the end :" + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
[
|
||||
{
|
||||
"paramName": "dbu",
|
||||
"paramLongName": "dataciteBaseURL",
|
||||
"paramDescription": "URL of Datacite Reports Endpoint",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "drp",
|
||||
"paramLongName": "dataciteReportPath",
|
||||
"paramDescription": "Path for Datacite Reports",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "dbhu",
|
||||
"paramLongName": "dbHiveUrl",
|
||||
"paramDescription": "activate tranform-only mode. Only apply transformation step",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "dbiu",
|
||||
"paramLongName": "dbImpalaUrl",
|
||||
"paramDescription": "activate tranform-only mode. Only apply transformation step",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "dusdbs",
|
||||
"paramLongName": "datasetUsageStatsDBSchema",
|
||||
"paramDescription": "activate tranform-only mode. Only apply transformation step",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "uspdbs",
|
||||
"paramLongName": "datasetsUsageStatsPermanentDBSchema",
|
||||
"paramDescription": "activate tranform-only mode. Only apply transformation step",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "sdbs",
|
||||
"paramLongName": "statsDBSchema",
|
||||
"paramDescription": "activate tranform-only mode. Only apply transformation step",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "rdbt",
|
||||
"paramLongName": "recreateDbAndTables",
|
||||
"paramDescription": "Re-create database and initial tables?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "pwed",
|
||||
"paramLongName": "datasetsEmptyDirs",
|
||||
"paramDescription": "Empty piwik directories?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "ftvi",
|
||||
"paramLongName": "finalTablesVisibleToImpala",
|
||||
"paramDescription": "Make the dataset_usage_stats, visible to Impala",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>${jobTracker}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>${nameNode}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>hiveMetastoreUris</name>
|
||||
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>hiveJdbcUrl</name>
|
||||
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>impalaJdbcUrl</name>
|
||||
<value>jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl;</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.wf.workflow.notification.url</name>
|
||||
<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
<workflow-app name="Datacite Datasets Usage Stats" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>hiveMetastoreUris</name>
|
||||
<description>Hive server metastore URIs</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hiveJdbcUrl</name>
|
||||
<description>Hive server jdbc url</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>impalaJdbcUrl</name>
|
||||
<description>Impala server jdbc url</description>
|
||||
</property>
|
||||
</parameters>
|
||||
|
||||
<global>
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>hive.metastore.uris</name>
|
||||
<value>${hiveMetastoreUris}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>mapreduce.job.queuename</name>
|
||||
<value>${queueName}</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.launcher.mapred.job.queue.name</name>
|
||||
<value>${oozieLauncherQueueName}</value>
|
||||
</property>
|
||||
</configuration>
|
||||
</global>
|
||||
|
||||
<start to="Step1"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
<action name='Step1'>
|
||||
<java>
|
||||
<main-class>eu.dnetlib.oa.graph.datasetsusagestats.export.ExecuteWorkflow</main-class>
|
||||
<arg>--dataciteBaseURL</arg>
|
||||
<arg>${dataciteBaseURL}</arg>
|
||||
<arg>--dataciteReportPath</arg>
|
||||
<arg>${dataciteReportPath}</arg>
|
||||
<arg>--dbHiveUrl</arg>
|
||||
<arg>${hiveJdbcUrl}</arg>
|
||||
<arg>--dbImpalaUrl</arg>
|
||||
<arg>${impalaJdbcUrl}</arg>
|
||||
<arg>--datasetUsageStatsDBSchema</arg>
|
||||
<arg>${datasetUsageStatsDBSchema}</arg>
|
||||
<arg>--datasetsUsageStatsPermanentDBSchema</arg>
|
||||
<arg>${datasetsUsageStatsPermanentDBSchema}</arg>
|
||||
<arg>--statsDBSchema</arg>
|
||||
<arg>${statsDBSchema}</arg>
|
||||
<arg>--recreateDbAndTables</arg>
|
||||
<arg>${recreateDbAndTables}</arg>
|
||||
<arg>--datasetsEmptyDirs</arg>
|
||||
<arg>${datasetsEmptyDirs}</arg>
|
||||
<arg>--finalTablesVisibleToImpala</arg>
|
||||
<arg>${finalTablesVisibleToImpala}</arg>
|
||||
<capture-output/>
|
||||
</java>
|
||||
<ok to="End" />
|
||||
<error to="Kill" />
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
@ -65,6 +65,8 @@ public class ExecuteWorkflow {
|
|||
|
||||
static int numberOfDownloadThreads;
|
||||
|
||||
static int b2SSHAREID;
|
||||
|
||||
public static void main(String args[]) throws Exception {
|
||||
|
||||
// Sending the logs to the console
|
||||
|
|
@ -196,6 +198,8 @@ public class ExecuteWorkflow {
|
|||
|
||||
numberOfDownloadThreads = Integer.parseInt(parser.get("numberOfDownloadThreads"));
|
||||
|
||||
b2SSHAREID = Integer.parseInt(parser.get("b2shareID"));
|
||||
|
||||
UsageStatsExporter usagestatsExport = new UsageStatsExporter();
|
||||
usagestatsExport.export();
|
||||
// usagestatsExport.createdDBWithTablesOnly();
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ public class PiwikDownloadLogs {
|
|||
ResultSet rs = statement
|
||||
.executeQuery(
|
||||
"SELECT distinct piwik_id from " + ConnectDB.getStatsDBSchema()
|
||||
+ ".datasource where piwik_id is not null and piwik_id <> 0 order by piwik_id");
|
||||
+ ".datasource where piwik_id is not null and piwik_id <> 0 and piwik_id <> 196 order by piwik_id");
|
||||
|
||||
// Getting all the piwikids in a list for logging reasons & limitting the list
|
||||
// to the max number of piwikids
|
||||
|
|
|
|||
|
|
@ -0,0 +1,204 @@
|
|||
|
||||
package eu.dnetlib.oa.graph.usagerawdata.export;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.json.simple.JSONArray;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author D. Pierrakos
|
||||
*/
|
||||
public class PiwikDownloadLogs_B2SHARE {
|
||||
|
||||
private final String piwikUrl;
|
||||
private Date startDate;
|
||||
private final String tokenAuth;
|
||||
|
||||
/*
|
||||
* The Piwik's API method
|
||||
*/
|
||||
private final String APImethod = "?module=API&method=Live.getLastVisitsDetails";
|
||||
private final String format = "&format=json";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PiwikDownloadLogs_B2SHARE.class);
|
||||
|
||||
public PiwikDownloadLogs_B2SHARE(String piwikUrl, String tokenAuth) {
|
||||
this.piwikUrl = piwikUrl;
|
||||
this.tokenAuth = tokenAuth;
|
||||
|
||||
}
|
||||
|
||||
private String getPiwikLogUrl() {
|
||||
return "https://" + piwikUrl + "/";
|
||||
}
|
||||
|
||||
private String getJson(String url) throws Exception {
|
||||
try {
|
||||
logger.debug("Connecting to download the JSON: " + url);
|
||||
URL website = new URL(url);
|
||||
URLConnection connection = website.openConnection();
|
||||
|
||||
StringBuilder response;
|
||||
try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
|
||||
response = new StringBuilder();
|
||||
String inputLine;
|
||||
while ((inputLine = in.readLine()) != null) {
|
||||
response.append(inputLine);
|
||||
}
|
||||
}
|
||||
return response.toString();
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to get URL: " + url + " Exception: " + e);
|
||||
throw new Exception("Failed to get URL: " + url + " Exception: " + e.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public void GetOpenAIREB2SHARELogs(String repoLogsPath) throws Exception {
|
||||
|
||||
Statement statement = ConnectDB.getHiveConnection().createStatement();
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
||||
|
||||
List<Integer> piwikIdToVisit = new ArrayList<Integer>();
|
||||
piwikIdToVisit.add(ExecuteWorkflow.b2SSHAREID);
|
||||
logger.info("B2SHARE piwikId for download: " + piwikIdToVisit);
|
||||
|
||||
if (ExecuteWorkflow.numberOfPiwikIdsToDownload > 0
|
||||
&& ExecuteWorkflow.numberOfPiwikIdsToDownload <= piwikIdToVisit.size()) {
|
||||
logger.info("Trimming piwikIds list to the size of: " + ExecuteWorkflow.numberOfPiwikIdsToDownload);
|
||||
piwikIdToVisit = piwikIdToVisit.subList(0, ExecuteWorkflow.numberOfPiwikIdsToDownload);
|
||||
}
|
||||
|
||||
logger.info("Downloading for the followins piwikIds: " + piwikIdToVisit);
|
||||
|
||||
// ExecutorService executor = Executors.newFixedThreadPool(ExecuteWorkflow.numberOfDownloadThreads);
|
||||
for (int siteId : piwikIdToVisit) {
|
||||
// Setting the starting period
|
||||
Calendar start = (Calendar) ExecuteWorkflow.startingLogPeriod.clone();
|
||||
logger.info("Starting period for log download: " + sdf.format(start.getTime()));
|
||||
|
||||
// Setting the ending period (last day of the month)
|
||||
// Calendar end = (Calendar) ExecuteWorkflow.endingLogPeriod.clone();
|
||||
Calendar end = Calendar.getInstance();
|
||||
end.add(Calendar.DAY_OF_MONTH, -1);
|
||||
// end.add(Calendar.MONTH, +1);
|
||||
// end.add(Calendar.DAY_OF_MONTH, -1);
|
||||
logger.info("Ending period for log download: " + sdf.format(end.getTime()));
|
||||
|
||||
logger.info("Now working on piwikId: " + siteId);
|
||||
|
||||
PreparedStatement st = ConnectDB.DB_HIVE_CONNECTION
|
||||
.prepareStatement(
|
||||
"SELECT max(timestamp) FROM " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".piwiklog WHERE source=?");
|
||||
st.setInt(1, siteId);
|
||||
Date dateMax = null;
|
||||
ResultSet rs_date = st.executeQuery();
|
||||
while (rs_date.next()) {
|
||||
logger.info("Found max date: " + rs_date.getString(1) + " for repository " + siteId);
|
||||
|
||||
if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null")
|
||||
&& !rs_date.getString(1).equals("")) {
|
||||
start.setTime(sdf.parse(rs_date.getString(1)));
|
||||
dateMax = sdf.parse(rs_date.getString(1));
|
||||
}
|
||||
}
|
||||
rs_date.close();
|
||||
|
||||
for (Calendar currDay = (Calendar) start.clone(); currDay.before(end); currDay.add(Calendar.DATE, 1)) {
|
||||
// logger.info("Date used " + currDay.toString());
|
||||
// Runnable worker = new WorkerThread(currDay, siteId, repoLogsPath, portalLogPath, portalMatomoID);
|
||||
// executor.execute(worker);// calling execute method of ExecutorService
|
||||
logger.info("Date used " + currDay.getTime().toString());
|
||||
|
||||
if (dateMax != null && currDay.getTime().compareTo(dateMax) <= 0) {
|
||||
logger.info("Date found in logs " + dateMax + " and not downloanding Matomo logs for " + siteId);
|
||||
} else {
|
||||
GetOpenAIRELogsB2SHAREForDate(currDay, siteId, repoLogsPath);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
// executor.shutdown();
|
||||
// while (!executor.isTerminated()) {
|
||||
// }
|
||||
// System.out.println("Finished all threads");
|
||||
}
|
||||
|
||||
public void GetOpenAIRELogsB2SHAREForDate(Calendar currDay, int siteId, String repoLogsPath) throws Exception {
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
||||
|
||||
Date date = currDay.getTime();
|
||||
logger.info("Downloading logs for repoid " + siteId + " and for " + sdf.format(date));
|
||||
|
||||
String period = "&period=day&date=" + sdf.format(date);
|
||||
String outFolder = repoLogsPath;
|
||||
|
||||
String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + siteId + period + format
|
||||
+ "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;
|
||||
String content = "";
|
||||
|
||||
int i = 0;
|
||||
|
||||
JSONParser parser = new JSONParser();
|
||||
StringBuffer totalContent = new StringBuffer();
|
||||
FileSystem fs = FileSystem.get(new Configuration());
|
||||
|
||||
do {
|
||||
int writtenBytes = 0;
|
||||
String apiUrl = baseApiUrl;
|
||||
|
||||
if (i > 0) {
|
||||
apiUrl += "&filter_offset=" + (i * 1000);
|
||||
}
|
||||
|
||||
content = getJson(apiUrl);
|
||||
if (content.length() == 0 || content.equals("[]")) {
|
||||
break;
|
||||
}
|
||||
|
||||
FSDataOutputStream fin = fs
|
||||
.create(
|
||||
new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) + "_offset_" + i
|
||||
+ ".json"),
|
||||
true);
|
||||
JSONArray jsonArray = (JSONArray) parser.parse(content);
|
||||
for (Object aJsonArray : jsonArray) {
|
||||
JSONObject jsonObjectRaw = (JSONObject) aJsonArray;
|
||||
byte[] jsonObjectRawBytes = jsonObjectRaw.toJSONString().getBytes();
|
||||
fin.write(jsonObjectRawBytes);
|
||||
fin.writeChar('\n');
|
||||
|
||||
writtenBytes += jsonObjectRawBytes.length + 1;
|
||||
}
|
||||
|
||||
fin.close();
|
||||
System.out
|
||||
.println(
|
||||
Thread.currentThread().getName() + " (Finished writing) Wrote " + writtenBytes
|
||||
+ " bytes. Filename: " + siteId + "_Piwiklog" + sdf.format((date)) + "_offset_" + i
|
||||
+ ".json");
|
||||
|
||||
i++;
|
||||
} while (true);
|
||||
|
||||
fs.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -179,6 +179,10 @@ public class PiwikStatsDB {
|
|||
createPedocsOldUsageData();
|
||||
logger.info("Pedocs Tables Created");
|
||||
|
||||
logger.info("Create Datacite Tables");
|
||||
createDatasetsUsageData();
|
||||
logger.info("Datacite Tables Created");
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to process logs: " + e);
|
||||
throw new Exception("Failed to process logs: " + e.toString(), e);
|
||||
|
|
@ -281,6 +285,7 @@ public class PiwikStatsDB {
|
|||
|
||||
// clean view double clicks
|
||||
logger.info("Cleaning action double clicks");
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp "
|
||||
+ "WHERE EXISTS (\n"
|
||||
+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
|
||||
|
|
@ -750,6 +755,16 @@ public class PiwikStatsDB {
|
|||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped sarc_sushilogtmp_json_non_array");
|
||||
|
||||
logger.info("Dropping piwiklogb2sharetmp");
|
||||
sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped piwiklogb2sharetmp");
|
||||
|
||||
logger.info("Dropping piwiklog_b2share_tmp_json");
|
||||
sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog_b2share_tmp_json";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped piwiklog_b2share_tmp_json");
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getHiveConnection().close();
|
||||
|
||||
|
|
@ -832,4 +847,32 @@ public class PiwikStatsDB {
|
|||
logger.info("PeDocs Old Downloads Table created");
|
||||
|
||||
}
|
||||
|
||||
public void createDatasetsUsageData() throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Dropping datacite_views");
|
||||
String sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped datacite_views");
|
||||
|
||||
logger.info("Dropping datacite_downloads");
|
||||
sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Dropped datacite_downloads");
|
||||
|
||||
logger.info("Creating Datasets Views Table");
|
||||
sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".datacite_views as select * from openaire_prod_datacite_usage_stats.datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datasets Views Table created");
|
||||
|
||||
logger.info("Creating Datasets Downloads Table");
|
||||
sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".datacite_downloads as select * from openaire_prod_datacite_usage_stats.datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datasets Downloads Table created");
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,304 @@
|
|||
|
||||
package eu.dnetlib.oa.graph.usagerawdata.export;
|
||||
|
||||
import java.io.*;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author D. Pierrakos, S. Zoupanos
|
||||
*/
|
||||
public class PiwikStatsDB_B2SHARE {
|
||||
|
||||
private String logPath;
|
||||
private String logRepoPath;
|
||||
private String logPortalPath;
|
||||
|
||||
private Statement stmt = null;
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(PiwikStatsDB_B2SHARE.class);
|
||||
|
||||
private String CounterRobotsURL;
|
||||
private ArrayList robotsList;
|
||||
|
||||
public PiwikStatsDB_B2SHARE(String logRepoPath, String logPortalPath) throws Exception {
|
||||
this.logRepoPath = logRepoPath;
|
||||
this.logPortalPath = logPortalPath;
|
||||
|
||||
}
|
||||
|
||||
public ArrayList getRobotsList() {
|
||||
return robotsList;
|
||||
}
|
||||
|
||||
public void setRobotsList(ArrayList robotsList) {
|
||||
this.robotsList = robotsList;
|
||||
}
|
||||
|
||||
public String getCounterRobotsURL() {
|
||||
return CounterRobotsURL;
|
||||
}
|
||||
|
||||
public void setCounterRobotsURL(String CounterRobotsURL) {
|
||||
this.CounterRobotsURL = CounterRobotsURL;
|
||||
}
|
||||
|
||||
public void processB2SHARELogs() throws Exception {
|
||||
try {
|
||||
|
||||
logger.info("Processing B2SHARE logs");
|
||||
processLog();
|
||||
logger.info("B2SHARE logs process done");
|
||||
|
||||
logger.info("Removing double clicks from B2SHARE logs");
|
||||
removeDoubleClicks();
|
||||
logger.info("Removing double clicks from B2SHARE logs done");
|
||||
|
||||
logger.info("Updating Production Tables");
|
||||
updateProdTables();
|
||||
logger.info("Updated Production Tables");
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to process logs: " + e);
|
||||
throw new Exception("Failed to process logs: " + e.toString(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public void processLog() throws Exception {
|
||||
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Adding JSON Serde jar");
|
||||
stmt.executeUpdate("add jar /usr/share/cmf/common_jars/hive-hcatalog-core-1.1.0-cdh5.14.0.jar");
|
||||
logger.info("Added JSON Serde jar");
|
||||
|
||||
logger.info("Dropping piwiklog_b2share_tmp_json table");
|
||||
String drop_piwiklogtmp_json = "DROP TABLE IF EXISTS "
|
||||
+ ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".piwiklog_b2share_tmp_json";
|
||||
stmt.executeUpdate(drop_piwiklogtmp_json);
|
||||
logger.info("Dropped piwiklog_b2share_tmp_json table");
|
||||
|
||||
logger.info("Creating piwiklog_b2share_tmp_json");
|
||||
String create_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS "
|
||||
+ ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".piwiklog_b2share_tmp_json(\n"
|
||||
+ " `idSite` STRING,\n"
|
||||
+ " `idVisit` STRING,\n"
|
||||
+ " `country` STRING,\n"
|
||||
+ " `referrerName` STRING,\n"
|
||||
+ " `browser` STRING,\n"
|
||||
+ " `actionDetails` ARRAY<\n"
|
||||
+ " struct<\n"
|
||||
+ " type: STRING,\n"
|
||||
+ " url: STRING,\n"
|
||||
+ " eventAction: STRING,\n"
|
||||
+ " eventName: STRING,\n"
|
||||
+ " timestamp: String\n"
|
||||
+ " >\n"
|
||||
+ " >\n"
|
||||
+ ")\n"
|
||||
+ "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n"
|
||||
+ "LOCATION '" + ExecuteWorkflow.repoLogPath + "'\n"
|
||||
+ "TBLPROPERTIES (\"transactional\"=\"false\")";
|
||||
stmt.executeUpdate(create_piwiklogtmp_json);
|
||||
logger.info("Created piwiklog_b2share_tmp_json");
|
||||
|
||||
logger.info("Dropping piwiklogtmp table");
|
||||
String drop_piwiklogtmp = "DROP TABLE IF EXISTS "
|
||||
+ ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".piwiklogtmp";
|
||||
stmt.executeUpdate(drop_piwiklogtmp);
|
||||
logger.info("Dropped piwiklogtmp");
|
||||
|
||||
logger.info("Creating piwiklogb2sharetmp");
|
||||
String create_piwiklogtmp = "CREATE TABLE "
|
||||
+ ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".piwiklogb2sharetmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, "
|
||||
+ "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) "
|
||||
+ "clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true')";
|
||||
stmt.executeUpdate(create_piwiklogtmp);
|
||||
logger.info("Created piwiklogb2sharetmp");
|
||||
|
||||
logger.info("Inserting into piwiklogb2sharetmp");
|
||||
String insert_piwiklogtmp = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
|
||||
+ "SELECT DISTINCT cast(idSite as BIGINT) as source, idVisit as id_Visit, country, "
|
||||
+ "actiondetail.eventAction as action, actiondetail.url as url, "
|
||||
+ "actiondetail.eventName as entity_id, "
|
||||
+ "'repItem' as source_item_type, from_unixtime(cast(actiondetail.timestamp as BIGINT)) as timestamp, "
|
||||
+ "referrerName as referrer_name, browser as agent\n"
|
||||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog_b2share_tmp_json\n"
|
||||
+ "LATERAL VIEW explode(actiondetails) actiondetailsTable AS actiondetail";
|
||||
stmt.executeUpdate(insert_piwiklogtmp);
|
||||
logger.info("Inserted into piwiklogb2sharetmp");
|
||||
|
||||
stmt.close();
|
||||
}
|
||||
|
||||
public void removeDoubleClicks() throws Exception {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Cleaning download double clicks");
|
||||
// clean download double clicks
|
||||
String sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
|
||||
+ "WHERE EXISTS (\n"
|
||||
+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
|
||||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p1, "
|
||||
+ ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p2\n"
|
||||
+ "WHERE p1.source=p2.source AND p1.id_visit=p2.id_visit AND p1.entity_id=p2.entity_id \n"
|
||||
+ "AND p1.action=p2.action AND p1.action='download' AND p1.timestamp!=p2.timestamp \n"
|
||||
+ "AND p1.timestamp<p2.timestamp AND ((unix_timestamp(p2.timestamp)-unix_timestamp(p1.timestamp))/60)<30 \n"
|
||||
+ "AND piwiklogb2sharetmp.source=p1.source AND piwiklogb2sharetmp.id_visit=p1.id_visit \n"
|
||||
+ "AND piwiklogb2sharetmp.action=p1.action AND piwiklogb2sharetmp.entity_id=p1.entity_id AND piwiklogb2sharetmp.timestamp=p1.timestamp)";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Cleaned download double clicks");
|
||||
|
||||
// clean view double clicks
|
||||
logger.info("Cleaning action double clicks");
|
||||
sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
|
||||
+ "WHERE EXISTS (\n"
|
||||
+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
|
||||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p1, "
|
||||
+ ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p2\n"
|
||||
+ "WHERE p1.source=p2.source AND p1.id_visit=p2.id_visit AND p1.entity_id=p2.entity_id \n"
|
||||
+ "AND p1.action=p2.action AND p1.action='action' AND p1.timestamp!=p2.timestamp \n"
|
||||
+ "AND p1.timestamp<p2.timestamp AND (unix_timestamp(p2.timestamp)-unix_timestamp(p1.timestamp))<10 \n"
|
||||
+ "AND piwiklogb2sharetmp.source=p1.source AND piwiklogb2sharetmp.id_visit=p1.id_visit \n"
|
||||
+ "AND piwiklogb2sharetmp.action=p1.action AND piwiklogb2sharetmp.entity_id=p1.entity_id AND piwiklogb2sharetmp.timestamp=p1.timestamp)";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Cleaned action double clicks");
|
||||
stmt.close();
|
||||
}
|
||||
|
||||
private void updateProdTables() throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Updating recordview to action piwiklog");
|
||||
String sqlUpdateAction = "UPDATE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
|
||||
+ "set action='action' where action='recordview'";
|
||||
stmt.executeUpdate(sqlUpdateAction);
|
||||
|
||||
logger.info("Updating fileDownload to download piwiklog");
|
||||
String sqlUpdateDownload = "UPDATE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
|
||||
+ "set action='download' where action='filedownload'";
|
||||
stmt.executeUpdate(sqlUpdateDownload);
|
||||
|
||||
logger.info("Inserting B2SHARE data to piwiklog");
|
||||
String sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp";
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
stmt.close();
|
||||
ConnectDB.getHiveConnection().close();
|
||||
|
||||
}
|
||||
|
||||
private ArrayList<String> listHdfsDir(String dir) throws Exception {
|
||||
|
||||
FileSystem hdfs = FileSystem.get(new Configuration());
|
||||
RemoteIterator<LocatedFileStatus> Files;
|
||||
ArrayList<String> fileNames = new ArrayList<>();
|
||||
|
||||
try {
|
||||
Path exportPath = new Path(hdfs.getUri() + dir);
|
||||
Files = hdfs.listFiles(exportPath, false);
|
||||
while (Files.hasNext()) {
|
||||
String fileName = Files.next().getPath().toString();
|
||||
fileNames.add(fileName);
|
||||
}
|
||||
|
||||
hdfs.close();
|
||||
} catch (Exception e) {
|
||||
logger.error("HDFS file path with exported data does not exist : " + new Path(hdfs.getUri() + logPath));
|
||||
throw new Exception("HDFS file path with exported data does not exist : " + logPath, e);
|
||||
}
|
||||
|
||||
return fileNames;
|
||||
}
|
||||
|
||||
private String readHDFSFile(String filename) throws Exception {
|
||||
String result;
|
||||
try {
|
||||
|
||||
FileSystem fs = FileSystem.get(new Configuration());
|
||||
// log.info("reading file : " + filename);
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(filename))));
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String line = br.readLine();
|
||||
|
||||
while (line != null) {
|
||||
if (!line.equals("[]")) {
|
||||
sb.append(line);
|
||||
}
|
||||
// sb.append(line);
|
||||
line = br.readLine();
|
||||
}
|
||||
result = sb.toString().replace("][{\"idSite\"", ",{\"idSite\"");
|
||||
if (result.equals("")) {
|
||||
result = "[]";
|
||||
}
|
||||
|
||||
// fs.close();
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage());
|
||||
throw new Exception(e);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private Connection getConnection() throws SQLException {
|
||||
return ConnectDB.getHiveConnection();
|
||||
}
|
||||
|
||||
public void createPedocsOldUsageData() throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Creating PeDocs Old Views Table");
|
||||
String sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".pedocsoldviews as select * from default.pedocsviews";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("PeDocs Old Views Table created");
|
||||
|
||||
logger.info("Creating PeDocs Old Downloads Table");
|
||||
sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".pedocsolddownloads as select * from default.pedocsdownloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("PeDocs Old Downloads Table created");
|
||||
|
||||
}
|
||||
|
||||
public void createDatasetsUsageData() throws SQLException {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
logger.info("Creating Datasets Views Table");
|
||||
String sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".datacite_views as select * from datasetsusagestats_20210301.datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datasets Views Table created");
|
||||
|
||||
logger.info("Creating Datasets Downloads Table");
|
||||
sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".datacite_downloads as select * from datasetsusagestats_20210301.datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datasets Downloads Table created");
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -142,8 +142,20 @@ public class UsageStatsExporter {
|
|||
sarcStats.updateSarcLogs();
|
||||
}
|
||||
logger.info("Sarc done");
|
||||
// finalize usagestats
|
||||
|
||||
PiwikDownloadLogs_B2SHARE b2sharePiwikID = new PiwikDownloadLogs_B2SHARE(ExecuteWorkflow.matomoBaseURL,
|
||||
ExecuteWorkflow.matomoAuthToken);
|
||||
b2sharePiwikID.GetOpenAIREB2SHARELogs(ExecuteWorkflow.repoLogPath);
|
||||
logger.info("B2SHARE done");
|
||||
|
||||
PiwikStatsDB_B2SHARE piwikstatsB2SHAREdb = new PiwikStatsDB_B2SHARE(ExecuteWorkflow.repoLogPath,
|
||||
ExecuteWorkflow.portalLogPath);
|
||||
piwikstatsB2SHAREdb.setCounterRobotsURL(cRobotsUrl);
|
||||
|
||||
logger.info("Processing B2SHARE logs");
|
||||
piwikstatsB2SHAREdb.processB2SHARELogs();
|
||||
|
||||
// finalize usagestats
|
||||
logger.info("Dropping tmp tables");
|
||||
if (ExecuteWorkflow.finalizeStats) {
|
||||
piwikstatsdb.finalizeStats();
|
||||
|
|
@ -161,6 +173,7 @@ public class UsageStatsExporter {
|
|||
piwikstatsdb.recreateDBAndTables();
|
||||
|
||||
piwikstatsdb.createPedocsOldUsageData();
|
||||
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger.info("Creating LaReferencia tables");
|
||||
|
|
|
|||
|
|
@ -215,5 +215,11 @@
|
|||
"paramLongName": "numberOfDownloadThreads",
|
||||
"paramDescription": "Number of download threads",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "b2shareID",
|
||||
"paramLongName": "b2shareID",
|
||||
"paramDescription": "B2SHARE Matomo ID",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<workflow-app name="Usage Graph Stats" xmlns="uri:oozie:workflow:0.5">
|
||||
<workflow-app name="Usage Activity Raw Data" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>hiveMetastoreUris</name>
|
||||
|
|
@ -78,6 +78,7 @@
|
|||
<arg>--sarcNumberOfIssnToDownload</arg><arg>${sarcNumberOfIssnToDownload}</arg>
|
||||
<arg>--finalizeStats</arg><arg>${finalizeStats}</arg>
|
||||
<arg>--numberOfDownloadThreads</arg><arg>${numberOfDownloadThreads}</arg>
|
||||
<arg>--b2shareID</arg><arg>${b2shareID}</arg>
|
||||
<capture-output/>
|
||||
</java>
|
||||
<ok to="End" />
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ public abstract class ConnectDB {
|
|||
Date today = Calendar.getInstance().getTime();
|
||||
String todayAsString = df.format(today);
|
||||
|
||||
return ConnectDB.usageStatsDBSchema + "_" + todayAsString;
|
||||
return ConnectDB.usageStatsDBSchema + todayAsString;
|
||||
}
|
||||
|
||||
public static String getStatsDBSchema() {
|
||||
|
|
|
|||
|
|
@ -35,20 +35,20 @@ public class PiwikStatsDB {
|
|||
|
||||
private void createDatabase() throws Exception {
|
||||
|
||||
// try {
|
||||
//
|
||||
// stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
//
|
||||
// logger.info("Dropping usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
|
||||
// String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + " CASCADE";
|
||||
// stmt.executeUpdate(dropDatabase);
|
||||
// } catch (Exception e) {
|
||||
// logger.error("Failed to drop database: " + e);
|
||||
// throw new Exception("Failed to drop database: " + e.toString(), e);
|
||||
// }
|
||||
//
|
||||
try {
|
||||
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
||||
logger.info("Dropping usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
|
||||
String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + " CASCADE";
|
||||
stmt.executeUpdate(dropDatabase);
|
||||
} catch (Exception e) {
|
||||
logger.error("Failed to drop database: " + e);
|
||||
throw new Exception("Failed to drop database: " + e.toString(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
logger.info("Creating usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
|
||||
String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema();
|
||||
stmt.executeUpdate(createDatabase);
|
||||
|
|
@ -132,7 +132,7 @@ public class PiwikStatsDB {
|
|||
+ "max(views) AS count, max(openaire_referrer) AS openaire "
|
||||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_views_monthly_tmp p, "
|
||||
+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
|
||||
+ "WHERE p.source=d.piwik_id AND p.id=ro.oid AND ro.oid!='200' "
|
||||
+ "WHERE p.source=d.piwik_id AND p.id=ro.oid AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
|
||||
+ "GROUP BY d.id, ro.id, month "
|
||||
+ "ORDER BY d.id, ro.id, month ";
|
||||
stmt.executeUpdate(create_views_stats);
|
||||
|
|
@ -145,7 +145,7 @@ public class PiwikStatsDB {
|
|||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_views_monthly_tmp p, "
|
||||
+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
|
||||
+ "WHERE p.source=" + ExecuteWorkflow.portalMatomoID
|
||||
+ " AND p.source=d.piwik_id and p.id=ro.id AND ro.oid!='200' "
|
||||
+ " AND p.source=d.piwik_id and p.id=ro.id AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
|
||||
+ "GROUP BY d.id, ro.id, month "
|
||||
+ "ORDER BY d.id, ro.id, month ";
|
||||
stmt.executeUpdate(create_pageviews_stats);
|
||||
|
|
@ -194,7 +194,7 @@ public class PiwikStatsDB {
|
|||
+ "max(downloads) AS count, max(openaire_referrer) AS openaire "
|
||||
+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_downloads_monthly_tmp p, "
|
||||
+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
|
||||
+ "WHERE p.source=d.piwik_id and p.id=ro.oid AND ro.oid!='200' "
|
||||
+ "WHERE p.source=d.piwik_id and p.id=ro.oid AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
|
||||
+ "GROUP BY d.id, ro.id, month "
|
||||
+ "ORDER BY d.id, ro.id, month ";
|
||||
stmt.executeUpdate(sql);
|
||||
|
|
@ -337,6 +337,96 @@ public class PiwikStatsDB {
|
|||
|
||||
}
|
||||
|
||||
public void uploadB2SHAREStats() throws Exception {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
||||
// Dropping B2SHARE b2share_result_views_monthly_tmp view
|
||||
logger.info("Dropping B2SHARE b2share_result_views_monthly_tmp view");
|
||||
String sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp";
|
||||
logger.info("Dropped b2share_result_views_monthly_tmp view ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
// Dropping B2SHARE b2share_result_views_monthly_tmp view
|
||||
logger.info("Dropping b2SHARE b2share_result_downloads_monthly_tmp view");
|
||||
sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp";
|
||||
logger.info("Dropped b2share_result_downloads_monthly_tmp view ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
// Dropping B2SHARE b2share_views_stats_tmp table
|
||||
logger.info("Dropping B2SHARE b2share_views_stats_tmp table");
|
||||
sql = "DROP TABLE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp";
|
||||
logger.info("Dropped b2share_views_stats_tmp table ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
// Dropping B2SHARE b2share_downloads_stats_tmp table
|
||||
logger.info("Dropping B2SHARE b2share_downloads_stats_tmp table");
|
||||
sql = "DROP TABLE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp";
|
||||
logger.info("Dropped b2share_downloads_stats_tmp table ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
// Creating B2SHARE b2share_result_views_monthly_tmp view
|
||||
logger.info("Creating B2SHARE b2share_result_views_monthly_tmp view");
|
||||
sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp "
|
||||
+ "AS SELECT entity_id, reflect('java.net.URLDecoder', 'decode', entity_id) AS id, "
|
||||
+ "COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, "
|
||||
+ "CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source "
|
||||
+ "FROM " + ConnectDB.getUsageRawDataDBSchema() + ".piwiklog "
|
||||
+ "WHERE action='action' and (source_item_type='oaItem' or source_item_type='repItem') and source=412 "
|
||||
+ "GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), source ORDER BY source, entity_id";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created b2share_result_views_monthly_tmp view ");
|
||||
|
||||
// Creating B2SHARE b2share_views_stats_tmp table
|
||||
logger.info("Creating B2SHARE b2share_views_stats_tmp table");
|
||||
sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp AS "
|
||||
+ "SELECT 'B2SHARE' as source, d.id as repository_id, ro.id as result_id, month as date, "
|
||||
+ "max(views) AS count, max(openaire_referrer) AS openaire FROM " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".b2share_result_views_monthly_tmp p, "
|
||||
+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
|
||||
+ "WHERE p.id=ro.oid and d.id='re3data_____::ad3609c351bd520edf6f10f5e0d9b877' "
|
||||
+ "GROUP BY d.id, ro.id, month ORDER BY d.id, ro.id";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created B2SHARE b2share_views_stats_tmp table");
|
||||
|
||||
// Creating B2SHARE b2share_result_downloads_monthly_tmp view
|
||||
logger.info("Creating B2SHARE b2share_result_downloads_monthly_tmp view");
|
||||
sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp "
|
||||
+ "AS SELECT entity_id, reflect('java.net.URLDecoder', 'decode', entity_id) AS id, "
|
||||
+ "COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, "
|
||||
+ "CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source "
|
||||
+ "FROM " + ConnectDB.getUsageRawDataDBSchema() + ".piwiklog "
|
||||
+ "WHERE action='download' and (source_item_type='oaItem' or source_item_type='repItem') and source=412 "
|
||||
+ "GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), source ORDER BY source, entity_id";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created b2share_result_downloads_monthly_tmp view ");
|
||||
|
||||
// Creating B2SHARE b2share_downloads_stats_tmp table
|
||||
logger.info("Creating B2SHARE b2share_downloads_stats_tmp table");
|
||||
sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp AS "
|
||||
+ "SELECT 'B2SHARE' as source, d.id as repository_id, ro.id as result_id, month as date, "
|
||||
+ "max(views) AS count, max(openaire_referrer) AS openaire FROM " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".b2share_result_downloads_monthly_tmp p, "
|
||||
+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
|
||||
+ "WHERE p.id=ro.oid and d.id='re3data_____::ad3609c351bd520edf6f10f5e0d9b877' "
|
||||
+ "GROUP BY d.id, ro.id, month ORDER BY d.id, ro.id";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Created B2SHARE b2share_downloads_stats_tmp table");
|
||||
|
||||
// Dropping B2SHARE b2share_result_views_monthly_tmp view
|
||||
logger.info("Dropping B2SHARE b2share_result_views_monthly_tmp view");
|
||||
sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp";
|
||||
logger.info("Dropped b2share_result_views_monthly_tmp view ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
// Dropping B2SHARE b2share_result_views_monthly_tmp view
|
||||
logger.info("Dropping B2SHARE b2share_result_downloads_monthly_tmp view");
|
||||
sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp";
|
||||
logger.info("Dropped b2share_result_downloads_monthly_tmp view ");
|
||||
stmt.executeUpdate(sql);
|
||||
|
||||
}
|
||||
|
||||
public void finalizeStats() throws Exception {
|
||||
stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
ConnectDB.getHiveConnection().setAutoCommit(false);
|
||||
|
|
@ -402,6 +492,13 @@ public class PiwikStatsDB {
|
|||
stmt.executeUpdate(sql);
|
||||
logger.info("LaReferencia views updated to views_stats");
|
||||
|
||||
// Inserting B2SHARE views stats
|
||||
logger.info("Inserting B2SHARE data to views_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".views_stats "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("B2SHARE views updated to views_stats");
|
||||
|
||||
logger.info("Creating downloads_stats table");
|
||||
String createDownloadsStats = "CREATE TABLE IF NOT EXISTS "
|
||||
+ ConnectDB.getUsageStatsDBSchema()
|
||||
|
|
@ -425,12 +522,18 @@ public class PiwikStatsDB {
|
|||
logger.info("Inserted Pedocs data to downloads_stats");
|
||||
|
||||
// Inserting TUDELFT downloads stats
|
||||
logger.info("Inserting TUDELFT old data to downloads_stats");
|
||||
logger.info("Inserting TUDELFT data to downloads_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".tudelft_downloads_stats_tmp";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Inserted TUDELFT data to downloads_stats");
|
||||
|
||||
// Inserting B2SHARE downloads stats
|
||||
logger.info("Inserting B2SHARE data to downloads_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Inserted B2SHARE data to downloads_stats");
|
||||
// Inserting Lareferencia downloads stats
|
||||
logger.info("Inserting LaReferencia data to downloads_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
|
||||
|
|
@ -452,6 +555,20 @@ public class PiwikStatsDB {
|
|||
stmt.executeUpdate(sql);
|
||||
logger.info("SARC-OJS downloads updated to downloads_stats");
|
||||
|
||||
// Inserting Datacite views stats
|
||||
logger.info("Inserting Datacite views to views_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".views_stats "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageRawDataDBSchema() + ".datacite_views";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datacite views updated to views_stats");
|
||||
|
||||
// Inserting Datacite downloads stats
|
||||
logger.info("Inserting Datacite downloads to downloads_stats");
|
||||
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
|
||||
+ "SELECT * FROM " + ConnectDB.getUsageRawDataDBSchema() + ".datacite_downloads";
|
||||
stmt.executeUpdate(sql);
|
||||
logger.info("Datacite downloads updated to downloads_stats");
|
||||
|
||||
logger.info("Creating pageviews_stats table");
|
||||
String create_pageviews_stats = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
|
||||
+ ".pageviews_stats "
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ public class UsageStatsExporter {
|
|||
logger.info("Processing TUDELFT Stats");
|
||||
piwikstatsdb.uploadTUDELFTStats();
|
||||
logger.info("Processing TUDELFT Stats Done");
|
||||
logger.info("Processing B2SHARE Stats");
|
||||
piwikstatsdb.uploadB2SHAREStats();
|
||||
logger.info("Processing B2SHARE Stats Done");
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<workflow-app name="Usage Graph Stats" xmlns="uri:oozie:workflow:0.5">
|
||||
<workflow-app name="Usage Stats Update" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>hiveMetastoreUris</name>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<actions>
|
||||
<action>
|
||||
<actionName>test</actionName>
|
||||
<packagings>
|
||||
<packaging>*</packaging>
|
||||
</packagings>
|
||||
<goals>
|
||||
<goal>test</goal>
|
||||
</goals>
|
||||
<properties>
|
||||
<skipTests>true</skipTests>
|
||||
</properties>
|
||||
</action>
|
||||
</actions>
|
||||
Loading…
Reference in New Issue