.gitignore

@@ -24,4 +24,3 @@
 spark-warehouse
 /**/job-override.properties
 /**/*.log
-

dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java

@@ -115,6 +115,8 @@ public class AuthorMerger {
 	}
 
 	public static String pidToComparableString(StructuredProperty pid) {
+		if (pid == null)
+			return "";
 		return (pid.getQualifier() != null
 			? pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""
 			: "")

dhp-workflows/dhp-aggregation/pom.xml

@@ -8,6 +8,37 @@
     </parent>
     <artifactId>dhp-aggregation</artifactId>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.0.1</version>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>initialize</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                </configuration>
+            </plugin>
+        </plugins>
+
+    </build>
+    
   
     <dependencies>
 
@@ -24,12 +55,6 @@
             <groupId>eu.dnetlib.dhp</groupId>
             <artifactId>dhp-common</artifactId>
             <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.sun.xml.bind</groupId>
-                    <artifactId>jaxb-core</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         
          <dependency>
@@ -37,6 +62,13 @@
             <artifactId>dhp-schemas</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-graph-mapper</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+
 
         <dependency>
             <groupId>net.sf.saxon</groupId>

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTransformation.scala

@@ -0,0 +1,544 @@
+package eu.dnetlib.dhp.actionmanager.datacite
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup
+import eu.dnetlib.dhp.schema.action.AtomicAction
+import eu.dnetlib.dhp.schema.common.ModelConstants
+import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, KeyValue, Oaf, OafMapperUtils, OtherResearchProduct, Publication, Qualifier, Relation, Result, Software, StructuredProperty, Dataset => OafDataset}
+import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, KeyValue, Oaf, OtherResearchProduct, Publication, Qualifier, Relation, Result, Software, StructuredProperty, Dataset => OafDataset}
+import eu.dnetlib.dhp.utils.DHPUtils
+import org.apache.commons.lang3.StringUtils
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST.{JField, JObject, JString}
+import org.json4s.jackson.JsonMethods.parse
+
+import java.nio.charset.CodingErrorAction
+import java.text.SimpleDateFormat
+import java.time.LocalDate
+import java.time.chrono.ThaiBuddhistDate
+import java.time.format.DateTimeFormatter
+import java.util.{Date, Locale}
+import java.util.regex.Pattern
+import scala.collection.JavaConverters._
+import scala.io.{Codec, Source}
+
+case class DataciteType(doi: String, timestamp: Long, isActive: Boolean, json: String) {}
+
+case class NameIdentifiersType(nameIdentifierScheme: Option[String], schemeUri: Option[String], nameIdentifier: Option[String]) {}
+
+case class CreatorType(nameType: Option[String], nameIdentifiers: Option[List[NameIdentifiersType]], name: Option[String], familyName: Option[String], givenName: Option[String], affiliation: Option[List[String]]) {}
+
+case class TitleType(title: Option[String], titleType: Option[String], lang: Option[String]) {}
+
+case class SubjectType(subject: Option[String], subjectScheme: Option[String]) {}
+
+case class DescriptionType(descriptionType: Option[String], description: Option[String]) {}
+
+case class FundingReferenceType(funderIdentifierType: Option[String], awardTitle: Option[String], awardUri: Option[String], funderName: Option[String], funderIdentifier: Option[String], awardNumber: Option[String]) {}
+
+case class DateType(date: Option[String], dateType: Option[String]) {}
+
+case class HostedByMapType(openaire_id: String, datacite_name: String, official_name: String, similarity: Option[Float]) {}
+
+object DataciteToOAFTransformation {
+  val UNKNOWN_REPOSITORY_ORIGINALID = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18"
+  val DATACITE_ID = "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254"
+
+  val DNET_DATACITE_DATE = "dnet:dataCite_date"
+
+  val DNET_DATACITE_TITLE = "dnet:dataCite_title"
+  val SYSIMPORT_ACTIONSET = "sysimport:actionset"
+  val DNET_PROVENANCE_ACTIONS = "dnet:provenanceActions"
+
+  val PROVENANCE_ACTION_SET_QUALIFIER: Qualifier = OafMapperUtils.qualifier(SYSIMPORT_ACTIONSET, SYSIMPORT_ACTIONSET, DNET_PROVENANCE_ACTIONS, DNET_PROVENANCE_ACTIONS)
+
+  val MAIN_TITLE_QUALIFIER:Qualifier =  OafMapperUtils.qualifier("main title","main title",DNET_DATACITE_TITLE,DNET_DATACITE_TITLE)
+
+  implicit val codec: Codec = Codec("UTF-8")
+  codec.onMalformedInput(CodingErrorAction.REPLACE)
+  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
+
+  val DOI_CLASS = "doi"
+  val SUBJ_CLASS = "keywords"
+
+
+  val j_filter: List[String] = {
+    val s = Source.fromInputStream(getClass.getResourceAsStream("datacite_filter")).mkString
+    s.lines.toList
+  }
+
+  val mapper = new ObjectMapper()
+  val unknown_repository: HostedByMapType = HostedByMapType(UNKNOWN_REPOSITORY_ORIGINALID, ModelConstants.UNKNOWN_REPOSITORY.getValue, ModelConstants.UNKNOWN_REPOSITORY.getValue, Some(1.0F))
+
+  val dataInfo: DataInfo = generateDataInfo("0.9")
+  val DATACITE_COLLECTED_FROM: KeyValue = OafMapperUtils.keyValue(DATACITE_ID, "Datacite")
+
+  val hostedByMap: Map[String, HostedByMapType] = {
+    val s = Source.fromInputStream(getClass.getResourceAsStream("hostedBy_map.json")).mkString
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json: org.json4s.JValue = parse(s)
+    json.extract[Map[String, HostedByMapType]]
+  }
+
+  val df_en: DateTimeFormatter = DateTimeFormatter.ofPattern("[MM-dd-yyyy][MM/dd/yyyy][dd-MM-yy][dd-MMM-yyyy][dd/MMM/yyyy][dd-MMM-yy][dd/MMM/yy][dd-MM-yy][dd/MM/yy][dd-MM-yyyy][dd/MM/yyyy][yyyy-MM-dd][yyyy/MM/dd]", Locale.ENGLISH)
+  val df_it: DateTimeFormatter = DateTimeFormatter.ofPattern("[dd-MM-yyyy][dd/MM/yyyy]", Locale.ITALIAN)
+
+  val funder_regex: List[(Pattern, String)] = List(
+    (Pattern.compile("(info:eu-repo/grantagreement/ec/h2020/)(\\d\\d\\d\\d\\d\\d)(.*)", Pattern.MULTILINE | Pattern.CASE_INSENSITIVE), "40|corda__h2020::"),
+    (Pattern.compile("(info:eu-repo/grantagreement/ec/fp7/)(\\d\\d\\d\\d\\d\\d)(.*)", Pattern.MULTILINE | Pattern.CASE_INSENSITIVE), "40|corda_______::")
+
+  )
+
+  val Date_regex: List[Pattern] = List(
+    //Y-M-D
+    Pattern.compile("(18|19|20)\\d\\d([- /.])(0[1-9]|1[012])\\2(0[1-9]|[12][0-9]|3[01])", Pattern.MULTILINE),
+    //M-D-Y
+    Pattern.compile("((0[1-9]|1[012])|([1-9]))([- /.])(0[1-9]|[12][0-9]|3[01])([- /.])(18|19|20)?\\d\\d", Pattern.MULTILINE),
+    //D-M-Y
+    Pattern.compile("(?:(?:31(/|-|\\.)(?:0?[13578]|1[02]|(?:Jan|Mar|May|Jul|Aug|Oct|Dec)))\\1|(?:(?:29|30)(/|-|\\.)(?:0?[1,3-9]|1[0-2]|(?:Jan|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))\\2))(?:(?:1[6-9]|[2-9]\\d)?\\d{2})|(?:29(/|-|\\.)(?:0?2|(?:Feb))\\3(?:(?:(?:1[6-9]|[2-9]\\d)?(?:0[48]|[2468][048]|[13579][26])|(?:(?:16|[2468][048]|[3579][26])00))))|(?:0?[1-9]|1\\d|2[0-8])(/|-|\\.)(?:(?:0?[1-9]|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep))|(?:1[0-2]|(?:Oct|Nov|Dec)))\\4(?:(?:1[6-9]|[2-9]\\d)?\\d{2})", Pattern.MULTILINE),
+    //Y
+    Pattern.compile("(19|20)\\d\\d", Pattern.MULTILINE)
+  )
+
+
+  def filter_json(json: String): Boolean = {
+    j_filter.exists(f => json.contains(f))
+  }
+
+  def toActionSet(item: Oaf): (String, String) = {
+    val mapper = new ObjectMapper()
+
+    item match {
+      case dataset: OafDataset =>
+        val a: AtomicAction[OafDataset] = new AtomicAction[OafDataset]
+        a.setClazz(classOf[OafDataset])
+        a.setPayload(dataset)
+        (dataset.getClass.getCanonicalName, mapper.writeValueAsString(a))
+      case publication: Publication =>
+        val a: AtomicAction[Publication] = new AtomicAction[Publication]
+        a.setClazz(classOf[Publication])
+        a.setPayload(publication)
+        (publication.getClass.getCanonicalName, mapper.writeValueAsString(a))
+      case software: Software =>
+        val a: AtomicAction[Software] = new AtomicAction[Software]
+        a.setClazz(classOf[Software])
+        a.setPayload(software)
+        (software.getClass.getCanonicalName, mapper.writeValueAsString(a))
+      case orp: OtherResearchProduct =>
+        val a: AtomicAction[OtherResearchProduct] = new AtomicAction[OtherResearchProduct]
+        a.setClazz(classOf[OtherResearchProduct])
+        a.setPayload(orp)
+        (orp.getClass.getCanonicalName, mapper.writeValueAsString(a))
+
+      case relation: Relation =>
+        val a: AtomicAction[Relation] = new AtomicAction[Relation]
+        a.setClazz(classOf[Relation])
+        a.setPayload(relation)
+        (relation.getClass.getCanonicalName, mapper.writeValueAsString(a))
+      case _ =>
+        null
+    }
+
+  }
+
+
+  def embargo_end(embargo_end_date: String): Boolean = {
+    val dt = LocalDate.parse(embargo_end_date, DateTimeFormatter.ofPattern("[yyyy-MM-dd]"))
+    val td = LocalDate.now()
+    td.isAfter(dt)
+  }
+
+
+  def extract_date(input: String): Option[String] = {
+    val d = Date_regex.map(pattern => {
+      val matcher = pattern.matcher(input)
+      if (matcher.find())
+        matcher.group(0)
+      else
+        null
+    }
+    ).find(s => s != null)
+
+    if (d.isDefined) {
+      val a_date = if (d.get.length == 4) s"01-01-${d.get}" else d.get
+      try {
+        return Some(LocalDate.parse(a_date, df_en).toString)
+      } catch {
+        case _: Throwable => try {
+          return Some(LocalDate.parse(a_date, df_it).toString)
+        } catch {
+          case _: Throwable =>
+            return None
+        }
+      }
+    }
+    d
+  }
+
+
+  def fix_thai_date(input:String, format:String) :String = {
+    try {
+      val a_date = LocalDate.parse(input,DateTimeFormatter.ofPattern(format))
+      val d = ThaiBuddhistDate.of(a_date.getYear, a_date.getMonth.getValue, a_date.getDayOfMonth)
+      LocalDate.from(d).toString
+    } catch {
+      case _: Throwable => ""
+    }
+  }
+
+  def getTypeQualifier(resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup): (Qualifier, Qualifier) = {
+    if (resourceType != null && resourceType.nonEmpty) {
+      val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceType)
+      if (typeQualifier != null)
+        return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
+    }
+    if (schemaOrg != null && schemaOrg.nonEmpty) {
+      val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, schemaOrg)
+      if (typeQualifier != null)
+        return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
+
+    }
+    if (resourceTypeGeneral != null && resourceTypeGeneral.nonEmpty) {
+      val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceTypeGeneral)
+      if (typeQualifier != null)
+        return (typeQualifier, vocabularies.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid))
+
+    }
+    null
+  }
+
+
+  def getResult(resourceType: String, resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup): Result = {
+    val typeQualifiers: (Qualifier, Qualifier) = getTypeQualifier(resourceType, resourceTypeGeneral, schemaOrg, vocabularies)
+    if (typeQualifiers == null)
+      return null
+    val i = new Instance
+    i.setInstancetype(typeQualifiers._1)
+    typeQualifiers._2.getClassname match {
+      case "dataset" =>
+        val r = new OafDataset
+        r.setInstance(List(i).asJava)
+        return r
+      case "publication" =>
+        val r = new Publication
+        r.setInstance(List(i).asJava)
+        return r
+      case "software" =>
+        val r = new Software
+        r.setInstance(List(i).asJava)
+        return r
+      case "other" =>
+        val r = new OtherResearchProduct
+        r.setInstance(List(i).asJava)
+        return r
+    }
+    null
+  }
+
+
+  def available_date(input: String): Boolean = {
+
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json: org.json4s.JValue = parse(input)
+    val l: List[String] = for {
+      JObject(dates) <- json \\ "dates"
+      JField("dateType", JString(dateTypes)) <- dates
+    } yield dateTypes
+
+    l.exists(p => p.equalsIgnoreCase("available"))
+
+  }
+
+
+  def OPEN_ACCESS_RIGHT = {
+      val result = new Qualifier
+      result.setClassid("OPEN")
+      result.setClassid("OPEN")
+      result.setSchemeid(ModelConstants.DNET_ACCESS_MODES)
+      result.setSchemename(ModelConstants.DNET_ACCESS_MODES)
+      result
+    }
+
+
+
+  /**
+   * As describe in ticket #6377
+   * when the result come from figshare we need to remove subject
+   * and set Access rights OPEN.
+   * @param r
+   */
+  def fix_figshare(r: Result): Unit = {
+
+    if (r.getInstance() != null) {
+      val hosted_by_figshare = r.getInstance().asScala.exists(i => i.getHostedby != null && "figshare".equalsIgnoreCase(i.getHostedby.getValue))
+      if (hosted_by_figshare) {
+        r.getInstance().asScala.foreach(i => i.setAccessright(OPEN_ACCESS_RIGHT))
+        val l: List[StructuredProperty] = List()
+        r.setSubject(l.asJava)
+      }
+    }
+
+
+  }
+
+  def generateOAFDate(dt: String, q: Qualifier): StructuredProperty = {
+    OafMapperUtils.structuredProperty(dt, q, null)
+  }
+
+  def generateRelation(sourceId: String, targetId: String, relClass: String, cf: KeyValue, di: DataInfo): Relation = {
+
+    val r = new Relation
+    r.setSource(sourceId)
+    r.setTarget(targetId)
+    r.setRelType(ModelConstants.RESULT_PROJECT)
+    r.setRelClass(relClass)
+    r.setSubRelType(ModelConstants.OUTCOME)
+    r.setCollectedfrom(List(cf).asJava)
+    r.setDataInfo(di)
+    r
+
+
+  }
+
+  def get_projectRelation(awardUri: String, sourceId: String): List[Relation] = {
+    val match_pattern = funder_regex.find(s => s._1.matcher(awardUri).find())
+
+    if (match_pattern.isDefined) {
+      val m = match_pattern.get._1
+      val p = match_pattern.get._2
+      val grantId = m.matcher(awardUri).replaceAll("$2")
+      val targetId = s"$p${DHPUtils.md5(grantId)}"
+      List(
+        generateRelation(sourceId, targetId, "isProducedBy", DATACITE_COLLECTED_FROM, dataInfo),
+        generateRelation(targetId, sourceId, "produces", DATACITE_COLLECTED_FROM, dataInfo)
+      )
+    }
+    else
+      List()
+
+  }
+
+
+  def generateOAF(input: String, ts: Long, dateOfCollection: Long, vocabularies: VocabularyGroup): List[Oaf] = {
+    if (filter_json(input))
+      return List()
+
+    implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
+    lazy val json = parse(input)
+
+    val resourceType = (json \ "attributes" \ "types" \ "resourceType").extractOrElse[String](null)
+    val resourceTypeGeneral = (json \ "attributes" \ "types" \ "resourceTypeGeneral").extractOrElse[String](null)
+    val schemaOrg = (json \ "attributes" \ "types" \ "schemaOrg").extractOrElse[String](null)
+
+    val doi = (json \ "attributes" \ "doi").extract[String]
+    if (doi.isEmpty)
+      return List()
+
+    //Mapping type based on vocabularies dnet:publication_resource and dnet:result_typologies
+    val result = getResult(resourceType, resourceTypeGeneral, schemaOrg, vocabularies)
+    if (result == null)
+      return List()
+
+
+    val doi_q = OafMapperUtils.qualifier("doi", "doi", ModelConstants.DNET_PID_TYPES, ModelConstants.DNET_PID_TYPES)
+    val pid = OafMapperUtils.structuredProperty(doi, doi_q, dataInfo)
+    result.setPid(List(pid).asJava)
+    result.setId(OafMapperUtils.createOpenaireId(50, s"datacite____::$doi", true))
+    result.setOriginalId(List(doi).asJava)
+
+    val d = new Date(dateOfCollection * 1000)
+    val ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US)
+
+
+    result.setDateofcollection(ISO8601FORMAT.format(d))
+    result.setDateoftransformation(ISO8601FORMAT.format(ts))
+    result.setDataInfo(dataInfo)
+
+    val creators = (json \\ "creators").extractOrElse[List[CreatorType]](List())
+
+
+    val authors = creators.zipWithIndex.map { case (c, idx) =>
+      val a = new Author
+      a.setFullname(c.name.orNull)
+      a.setName(c.givenName.orNull)
+      a.setSurname(c.familyName.orNull)
+      if (c.nameIdentifiers != null && c.nameIdentifiers.isDefined && c.nameIdentifiers.get != null) {
+        a.setPid(c.nameIdentifiers.get.map(ni => {
+          val q = if (ni.nameIdentifierScheme.isDefined) vocabularies.getTermAsQualifier(ModelConstants.DNET_PID_TYPES, ni.nameIdentifierScheme.get.toLowerCase()) else null
+          if (ni.nameIdentifier != null && ni.nameIdentifier.isDefined) {
+            OafMapperUtils.structuredProperty(ni.nameIdentifier.get, q, dataInfo)
+          }
+          else
+            null
+
+        }
+        )
+          .asJava)
+      }
+      if (c.affiliation.isDefined)
+        a.setAffiliation(c.affiliation.get.filter(af => af.nonEmpty).map(af => OafMapperUtils.field(af, dataInfo)).asJava)
+      a.setRank(idx + 1)
+      a
+    }
+
+
+    val titles: List[TitleType] = (json \\ "titles").extractOrElse[List[TitleType]](List())
+
+    result.setTitle(titles.filter(t => t.title.nonEmpty).map(t => {
+      if (t.titleType.isEmpty) {
+        OafMapperUtils.structuredProperty(t.title.get, MAIN_TITLE_QUALIFIER, null)
+      } else {
+        OafMapperUtils.structuredProperty(t.title.get, t.titleType.get, t.titleType.get, DNET_DATACITE_TITLE, DNET_DATACITE_TITLE, null)
+      }
+    }).asJava)
+
+    if (authors == null || authors.isEmpty || !authors.exists(a => a != null))
+      return List()
+    result.setAuthor(authors.asJava)
+
+    val dates = (json \\ "dates").extract[List[DateType]]
+    val publication_year = (json \\ "publicationYear").extractOrElse[String](null)
+
+    val i_date = dates
+      .filter(d => d.date.isDefined && d.dateType.isDefined)
+      .find(d => d.dateType.get.equalsIgnoreCase("issued"))
+      .map(d => extract_date(d.date.get))
+    val a_date: Option[String] = dates
+      .filter(d => d.date.isDefined && d.dateType.isDefined && d.dateType.get.equalsIgnoreCase("available"))
+      .map(d => extract_date(d.date.get))
+      .find(d => d != null && d.isDefined)
+      .map(d => d.get)
+
+    if (a_date.isDefined) {
+      if(doi.startsWith("10.14457"))
+        result.setEmbargoenddate(OafMapperUtils.field(fix_thai_date(a_date.get,"[yyyy-MM-dd]"), null))
+      else
+        result.setEmbargoenddate(OafMapperUtils.field(a_date.get, null))
+    }
+    if (i_date.isDefined && i_date.get.isDefined) {
+      if(doi.startsWith("10.14457")) {
+        result.setDateofacceptance(OafMapperUtils.field(fix_thai_date(i_date.get.get,"[yyyy-MM-dd]"), null))
+        result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(fix_thai_date(i_date.get.get,"[yyyy-MM-dd]"), null))
+      }
+      else {
+        result.setDateofacceptance(OafMapperUtils.field(i_date.get.get, null))
+        result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(i_date.get.get, null))
+      }
+    }
+    else if (publication_year != null) {
+      if(doi.startsWith("10.14457")) {
+        result.setDateofacceptance(OafMapperUtils.field(fix_thai_date(s"01-01-$publication_year","[dd-MM-yyyy]"), null))
+        result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(fix_thai_date(s"01-01-$publication_year","[dd-MM-yyyy]"), null))
+
+      } else {
+        result.setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null))
+        result.getInstance().get(0).setDateofacceptance(OafMapperUtils.field(s"01-01-$publication_year", null))
+      }
+    }
+
+
+    result.setRelevantdate(dates.filter(d => d.date.isDefined && d.dateType.isDefined)
+      .map(d => (extract_date(d.date.get), d.dateType.get))
+      .filter(d => d._1.isDefined)
+      .map(d => (d._1.get, vocabularies.getTermAsQualifier(DNET_DATACITE_DATE, d._2.toLowerCase())))
+      .filter(d => d._2 != null)
+      .map(d => generateOAFDate(d._1, d._2)).asJava)
+
+    val subjects = (json \\ "subjects").extract[List[SubjectType]]
+
+    result.setSubject(subjects.filter(s => s.subject.nonEmpty)
+      .map(s =>
+        OafMapperUtils.structuredProperty(s.subject.get, SUBJ_CLASS, SUBJ_CLASS, ModelConstants.DNET_SUBJECT_TYPOLOGIES, ModelConstants.DNET_SUBJECT_TYPOLOGIES, null)
+      ).asJava)
+
+
+    result.setCollectedfrom(List(DATACITE_COLLECTED_FROM).asJava)
+
+    val descriptions = (json \\ "descriptions").extract[List[DescriptionType]]
+
+    result.setDescription(
+      descriptions
+        .filter(d => d.description.isDefined).
+        map(d =>
+          OafMapperUtils.field(d.description.get, null)
+        ).filter(s => s != null).asJava)
+
+
+    val publisher = (json \\ "publisher").extractOrElse[String](null)
+    if (publisher != null)
+      result.setPublisher(OafMapperUtils.field(publisher, null))
+
+
+    val language: String = (json \\ "language").extractOrElse[String](null)
+
+    if (language != null)
+      result.setLanguage(vocabularies.getSynonymAsQualifier(ModelConstants.DNET_LANGUAGES, language))
+
+
+    val instance = result.getInstance().get(0)
+
+    val client = (json \ "relationships" \ "client" \\ "id").extractOpt[String]
+
+    val accessRights: List[String] = for {
+      JObject(rightsList) <- json \\ "rightsList"
+      JField("rightsUri", JString(rightsUri)) <- rightsList
+    } yield rightsUri
+
+    val aRights: Option[Qualifier] = accessRights.map(r => {
+      vocabularies.getSynonymAsQualifier(ModelConstants.DNET_ACCESS_MODES, r)
+    }).find(q => q != null).map(q => {
+      val a = new Qualifier
+      a.setClassid(q.getClassid)
+      a.setClassname(q.getClassname)
+      a.setSchemeid(q.getSchemeid)
+      a.setSchemename(q.getSchemename)
+      a
+    })
+
+
+    val access_rights_qualifier = if (aRights.isDefined) aRights.get else OafMapperUtils.qualifier(ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE, ModelConstants.DNET_ACCESS_MODES, ModelConstants.DNET_ACCESS_MODES)
+
+    if (client.isDefined) {
+      val hb = hostedByMap.getOrElse(client.get.toUpperCase(), unknown_repository)
+      instance.setHostedby(OafMapperUtils.keyValue(generateDSId(hb.openaire_id), hb.official_name))
+      instance.setCollectedfrom(DATACITE_COLLECTED_FROM)
+      instance.setUrl(List(s"https://dx.doi.org/$doi").asJava)
+      instance.setAccessright(access_rights_qualifier)
+      val license = accessRights
+        .find(r => r.startsWith("http") && r.matches(".*(/licenses|/publicdomain|unlicense\\.org/|/legal-and-data-protection-notices|/download/license|/open-government-licence).*"))
+      if (license.isDefined)
+        instance.setLicense(OafMapperUtils.field(license.get, null))
+    }
+
+    val awardUris: List[String] = for {
+      JObject(fundingReferences) <- json \\ "fundingReferences"
+      JField("awardUri", JString(awardUri)) <- fundingReferences
+    } yield awardUri
+
+    val relations: List[Relation] = awardUris.flatMap(a => get_projectRelation(a, result.getId)).filter(r => r != null)
+    fix_figshare(result)
+    if (relations != null && relations.nonEmpty) {
+      List(result) ::: relations
+    }
+    else
+      List(result)
+  }
+
+  def generateDataInfo(trust: String): DataInfo = {
+    val di = new DataInfo
+    di.setDeletedbyinference(false)
+    di.setInferred(false)
+    di.setInvisible(false)
+    di.setTrust(trust)
+    di.setProvenanceaction(PROVENANCE_ACTION_SET_QUALIFIER)
+    di
+  }
+
+  def generateDSId(input: String): String = {
+    val b = StringUtils.substringBefore(input, "::")
+    val a = StringUtils.substringAfter(input, "::")
+    s"10|$b::${DHPUtils.md5(a)}"
+  }
+
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/ExportActionSetJobNode.scala

@@ -0,0 +1,40 @@
+package eu.dnetlib.dhp.actionmanager.datacite
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import org.apache.hadoop.io.Text
+import org.apache.hadoop.io.compress.GzipCodec
+import org.apache.hadoop.mapred.SequenceFileOutputFormat
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.io.Source
+
+object ExportActionSetJobNode {
+
+  val log: Logger = LoggerFactory.getLogger(ExportActionSetJobNode.getClass)
+
+  def main(args: Array[String]): Unit = {
+    val conf = new SparkConf
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/exportDataset_parameters.json")).mkString)
+    parser.parseArgument(args)
+    val master = parser.get("master")
+    val sourcePath = parser.get("sourcePath")
+    val targetPath = parser.get("targetPath")
+
+    val spark: SparkSession = SparkSession.builder().config(conf)
+      .appName(ExportActionSetJobNode.getClass.getSimpleName)
+      .master(master)
+      .getOrCreate()
+    implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
+    implicit val tEncoder:Encoder[(String,String)] = Encoders.tuple(Encoders.STRING,Encoders.STRING)
+
+    spark.read.load(sourcePath).as[Oaf]
+      .map(o =>DataciteToOAFTransformation.toActionSet(o))
+      .filter(o => o!= null)
+      .rdd.map(s => (new Text(s._1), new Text(s._2))).saveAsHadoopFile(s"$targetPath", classOf[Text], classOf[Text], classOf[SequenceFileOutputFormat[Text,Text]], classOf[GzipCodec])
+
+
+  }
+
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/datacite/GenerateDataciteDatasetSpark.scala

@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.actionmanager.datacite
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import eu.dnetlib.dhp.utils.ISLookupClientFactory
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession}
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.io.Source
+
+object GenerateDataciteDatasetSpark {
+
+  val log: Logger = LoggerFactory.getLogger(GenerateDataciteDatasetSpark.getClass)
+
+  def main(args: Array[String]): Unit = {
+    val conf = new SparkConf
+    val parser = new ArgumentApplicationParser(Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json")).mkString)
+    parser.parseArgument(args)
+    val master = parser.get("master")
+    val sourcePath = parser.get("sourcePath")
+    val targetPath = parser.get("targetPath")
+    val isLookupUrl: String = parser.get("isLookupUrl")
+    log.info("isLookupUrl: {}", isLookupUrl)
+
+    val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
+    val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
+    val spark: SparkSession = SparkSession.builder().config(conf)
+      .appName(GenerateDataciteDatasetSpark.getClass.getSimpleName)
+      .master(master)
+      .getOrCreate()
+
+    implicit val resEncoder: Encoder[Oaf] = Encoders.kryo[Oaf]
+
+    import spark.implicits._
+
+    spark.read.load(sourcePath).as[DataciteType]
+      .filter(d => d.isActive)
+      .flatMap(d => DataciteToOAFTransformation.generateOAF(d.json, d.timestamp, d.timestamp, vocabularies))
+      .filter(d => d != null)
+      .write.mode(SaveMode.Overwrite).save(targetPath)
+  }
+}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/PrepareProjects.java

@@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-
 import eu.dnetlib.dhp.actionmanager.project.utils.CSVProject;
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
 import eu.dnetlib.dhp.common.HdfsSupport;
@@ -33,7 +32,6 @@ public class PrepareProjects {
 	private static final Logger log = LoggerFactory.getLogger(PrepareProgramme.class);
 	private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
-
 	public static void main(String[] args) throws Exception {
 
 		String jsonConfiguration = IOUtils

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/datacite_filter

@@ -0,0 +1,28 @@
+TUBYDI - Assistir Filmes e Series Online Grátis
+123Movies
+WATCH FULL MOVIE
+Movierulz
+Full Movie Online
+MOVIé WatcH
+The King of Staten Island 2020 Online For Free
+Watch Train to Busan 2 2020 online for free
+Sixth Sense Movie Novelization
+Film Complet streaming vf gratuit en ligne
+watch now free
+LIVE stream watch
+LIVE stream UFC
+RBC Heritage live stream
+MLBStreams Free
+NFL Live Stream
+Live Stream Free
+Royal Ascot 2020 Live Stream
+TV Shows Full Episodes Official
+FuboTV
+Gomovies
+Online Free Trial Access
+123watch
+DÜŞÜK HAPI
+Bebek Düşürme Yöntemleri
+WHATSAP İLETİŞİM
+Cytotec
+düşük hapı

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/exportDataset_parameters.json

@@ -0,0 +1,21 @@
+[
+  {
+    "paramName": "s",
+    "paramLongName": "sourcePath",
+    "paramDescription": "the source mdstore path",
+    "paramRequired": true
+  },
+
+  {
+    "paramName": "t",
+    "paramLongName": "targetPath",
+    "paramDescription": "the target mdstore path",
+    "paramRequired": true
+  },
+  {
+    "paramName": "m",
+    "paramLongName": "master",
+    "paramDescription": "the master name",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/generate_dataset_params.json

@@ -0,0 +1,26 @@
+[
+  {
+    "paramName": "s",
+    "paramLongName": "sourcePath",
+    "paramDescription": "the source mdstore path",
+    "paramRequired": true
+  },
+  {
+    "paramName": "t",
+    "paramLongName": "targetPath",
+    "paramDescription": "the target mdstore path",
+    "paramRequired": true
+  },
+  {
+    "paramName": "m",
+    "paramLongName": "master",
+    "paramDescription": "the master name",
+    "paramRequired": true
+  },
+  {
+    "paramName": "i",
+    "paramLongName": "isLookupUrl",
+    "paramDescription": "the isLookup URL",
+    "paramRequired": true
+  }
+]

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/oozie_app/config-default.xml

@@ -0,0 +1,23 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite/oozie_app/workflow.xml

@@ -0,0 +1,46 @@
+<workflow-app name="Import_Datacite_and_transform_to_OAF" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>mainPath</name>
+            <description>the working path of Datacite stores</description>
+        </property>
+        <property>
+            <name>isLookupUrl</name>
+            <description>The IS lookUp service endopoint</description>
+        </property>
+    </parameters>
+
+    <start to="TransformJob"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name="TransformJob">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>TransformJob</name>
+            <class>eu.dnetlib.dhp.actionmanager.datacite.GenerateDataciteDatasetSpark</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.sql.shuffle.partitions=3840
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${mainPath}/datacite_dump</arg>
+            <arg>--targetPath</arg><arg>${mainPath}/production/datacite_oaf</arg>
+            <arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
+            <arg>--master</arg><arg>yarn-cluster</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite_export/oozie_app/config-default.xml

@@ -0,0 +1,23 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>yarnRM</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>hdfs://nameservice1</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+
+    <property>
+        <name>oozie.launcher.mapreduce.user.classpath.first</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/datacite_export/oozie_app/workflow.xml

@@ -0,0 +1,46 @@
+<workflow-app name="Datacite_to_ActionSet_Workflow" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>sourcePath</name>
+            <description>the working path of Datacite stores</description>
+        </property>
+        <property>
+            <name>outputPath</name>
+            <description>the path of Datacite ActionSet</description>
+        </property>
+    </parameters>
+
+    <start to="ExportDataset"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+
+    <action name="ExportDataset">
+        <spark xmlns="uri:oozie:spark-action:0.2">
+            <master>yarn-cluster</master>
+            <mode>cluster</mode>
+            <name>ExportDataset</name>
+            <class>eu.dnetlib.dhp.actionmanager.datacite.ExportActionSetJobNode</class>
+            <jar>dhp-aggregation-${projectVersion}.jar</jar>
+            <spark-opts>
+                --executor-memory=${sparkExecutorMemory}
+                --executor-cores=${sparkExecutorCores}
+                --driver-memory=${sparkDriverMemory}
+                --conf spark.sql.shuffle.partitions=3840
+                --conf spark.extraListeners=${spark2ExtraListeners}
+                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
+                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
+                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
+            </spark-opts>
+            <arg>--sourcePath</arg><arg>${sourcePath}</arg>
+            <arg>--targetPath</arg><arg>${outputPath}</arg>
+            <arg>--master</arg><arg>yarn-cluster</arg>
+        </spark>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-aggregation/src/test/java/eu/dentlib/dhp/aggregation/AbstractVocabularyTest.java

@@ -0,0 +1,50 @@
+
+package eu.dentlib.dhp.aggregation;
+
+import static org.mockito.Mockito.lenient;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.commons.io.IOUtils;
+import org.mockito.Mock;
+
+import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+public abstract class AbstractVocabularyTest {
+
+	@Mock
+	protected ISLookUpService isLookUpService;
+
+	protected VocabularyGroup vocabularies;
+
+	public void setUpVocabulary() throws ISLookUpException, IOException {
+		lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs());
+
+		lenient()
+			.when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY))
+			.thenReturn(synonyms());
+		vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService);
+	}
+
+	private static List<String> vocs() throws IOException {
+		return IOUtils
+			.readLines(
+				Objects
+					.requireNonNull(
+						AbstractVocabularyTest.class.getResourceAsStream("/eu/dnetlib/dhp/vocabulary/terms.txt")));
+	}
+
+	private static List<String> synonyms() throws IOException {
+		return IOUtils
+			.readLines(
+				Objects
+					.requireNonNull(
+						AbstractVocabularyTest.class.getResourceAsStream("/eu/dnetlib/dhp/vocabulary/synonyms.txt")));
+	}
+
+}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/datacite/DataciteToOAFTest.scala

@@ -0,0 +1,43 @@
+package eu.dnetlib.dhp.actionmanager.datacite
+
+
+import eu.dentlib.dhp.aggregation.AbstractVocabularyTest
+import eu.dnetlib.dhp.schema.oaf.Oaf
+import org.junit.jupiter.api.extension.ExtendWith
+import org.junit.jupiter.api.{BeforeEach, Test}
+import org.mockito.junit.jupiter.MockitoExtension
+import org.codehaus.jackson.map.ObjectMapper
+
+import scala.io.Source
+
+@ExtendWith(Array(classOf[MockitoExtension]))
+class DataciteToOAFTest extends  AbstractVocabularyTest{
+
+
+  @BeforeEach
+  def setUp() :Unit = {
+
+    super.setUpVocabulary()
+  }
+
+  @Test
+  def testMapping() :Unit = {
+    val record =Source.fromInputStream(getClass.getResourceAsStream("datacite.json")).mkString
+
+
+
+    val mapper = new ObjectMapper()
+    val res:List[Oaf] =DataciteToOAFTransformation.generateOAF(record, 0L,0L, vocabularies )
+    println (mapper.defaultPrettyPrintingWriter().writeValueAsString(res.head))
+
+
+  }
+  @Test
+  def testDate():Unit = {
+
+    println(DataciteToOAFTransformation.fix_thai_date("01-01-2561","[dd-MM-yyyy]"))
+    println(DataciteToOAFTransformation.fix_thai_date("2561-01-01","[yyyy-MM-dd]"))
+
+  }
+
+}

dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/ConversionUtils.java

@@ -26,6 +26,7 @@ import eu.dnetlib.broker.objects.OaBrokerRelatedDatasource;
 import eu.dnetlib.broker.objects.OaBrokerRelatedPublication;
 import eu.dnetlib.broker.objects.OaBrokerRelatedSoftware;
 import eu.dnetlib.broker.objects.OaBrokerTypedValue;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.Author;
 import eu.dnetlib.dhp.schema.oaf.Dataset;
 import eu.dnetlib.dhp.schema.oaf.Datasource;
@@ -144,7 +145,7 @@ public class ConversionUtils {
 			.filter(pid -> pid != null)
 			.filter(pid -> pid.getQualifier() != null)
 			.filter(pid -> pid.getQualifier().getClassid() != null)
-			.filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase("orcid"))
+			.filter(pid -> pid.getQualifier().getClassid().equalsIgnoreCase(ModelConstants.ORCID))
 			.map(pid -> pid.getValue())
 			.map(pid -> cleanOrcid(pid))
 			.filter(StringUtils::isNotBlank)

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DatePicker.java

@@ -18,7 +18,7 @@ import eu.dnetlib.dhp.schema.oaf.Field;
 
 public class DatePicker {
 
-	private static final String DATE_PATTERN = "\\d{4}-\\d{2}-\\d{2}";
+	private static final String DATE_PATTERN = "^(\\d{4})-(\\d{2})-(\\d{2})";
 	private static final String DATE_DEFAULT_SUFFIX = "01-01";
 	private static final int YEAR_LB = 1300;
 	private static final int YEAR_UB = Year.now().getValue() + 5;
@@ -28,6 +28,7 @@ public class DatePicker {
 		final Map<String, Integer> frequencies = dateofacceptance
 			.parallelStream()
 			.filter(StringUtils::isNotBlank)
+			.map(d -> substringBefore(d, "T"))
 			.collect(Collectors.toConcurrentMap(w -> w, w -> 1, Integer::sum));
 
 		if (frequencies.isEmpty()) {

dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/DatePickerTest.java

@@ -0,0 +1,44 @@
+
+package eu.dnetlib.dhp.oa.dedup;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Collection;
+
+import org.junit.jupiter.api.Test;
+
+import com.clearspring.analytics.util.Lists;
+
+public class DatePickerTest {
+
+	Collection<String> dates = Lists.newArrayList();
+
+	@Test
+	public void testPickISO() {
+		dates.add("2016-01-01T12:00:00Z");
+		dates.add("2016-06-16T12:00:00Z");
+		dates.add("2020-01-01T12:00:00Z");
+		dates.add("2020-10-01T12:00:00Z");
+		assertEquals("2020-10-01", DatePicker.pick(dates).getValue());
+	}
+
+	@Test
+	public void testPickSimple() {
+		dates.add("2016-01-01");
+		dates.add("2016-06-16");
+		dates.add("2020-01-01");
+		dates.add("2020-10-01");
+		assertEquals("2020-10-01", DatePicker.pick(dates).getValue());
+	}
+
+	@Test
+	public void testPickFrequent() {
+		dates.add("2016-02-01");
+		dates.add("2016-02-01");
+		dates.add("2016-02-01");
+		dates.add("2020-10-01");
+		assertEquals("2016-02-01", DatePicker.pick(dates).getValue());
+	}
+
+}

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/DoiBoostMappingUtil.scala

@@ -5,6 +5,7 @@ import eu.dnetlib.dhp.schema.oaf.{DataInfo, Dataset, Field, Instance, KeyValue,
 import eu.dnetlib.dhp.utils.DHPUtils
 import org.apache.commons.lang3.StringUtils
 import com.fasterxml.jackson.databind.ObjectMapper
+import eu.dnetlib.dhp.schema.common.ModelConstants
 import org.json4s
 import org.json4s.DefaultFormats
 import org.json4s.jackson.JsonMethods.parse
@@ -28,7 +29,6 @@ object DoiBoostMappingUtil {
   //STATIC STRING
   val MAG = "microsoft"
   val MAG_NAME = "Microsoft Academic Graph"
-  val ORCID = "orcid"
   val ORCID_PENDING = "orcid_pending"
   val CROSSREF = "Crossref"
   val UNPAYWALL = "UnpayWall"
@@ -37,8 +37,6 @@ object DoiBoostMappingUtil {
   val doiBoostNSPREFIX = "doiboost____"
   val OPENAIRE_PREFIX = "openaire____"
   val SEPARATOR = "::"
-  val DNET_LANGUAGES = "dnet:languages"
-  val PID_TYPES = "dnet:pid_types"
 
   val invalidName = List(",", "none none", "none, none", "none &na;", "(:null)", "test test test", "test test", "test", "&na; &na;")
 
@@ -326,8 +324,8 @@ object DoiBoostMappingUtil {
   def createORIDCollectedFrom(): KeyValue = {
 
     val cf = new KeyValue
-    cf.setValue(ORCID)
+    cf.setValue(ModelConstants.ORCID_DS)
-    cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ORCID.toLowerCase))
+    cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + DHPUtils.md5(ModelConstants.ORCID))
     cf
 
   }

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala

@@ -87,7 +87,7 @@ case object Crossref2Oaf {
 
     //MAPPING Crossref DOI into PID
     val doi: String = (json \ "DOI").extract[String]
-    result.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava)
+    result.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
 
     //MAPPING Crossref DOI into OriginalId
     //and Other Original Identifier of dataset like clinical-trial-number

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/crossref/SparkMapDumpIntoOAF.scala

@@ -33,9 +33,9 @@ object SparkMapDumpIntoOAF {
     implicit val mapEncoderPubs: Encoder[Publication] = Encoders.kryo[Publication]
     implicit val mapEncoderRelatons: Encoder[Relation] = Encoders.kryo[Relation]
     implicit val mapEncoderDatasets: Encoder[oaf.Dataset] = Encoders.kryo[OafDataset]
+    implicit val mrEncoder: Encoder[CrossrefDT] = Encoders.kryo[CrossrefDT]
 
     val targetPath = parser.get("targetPath")
-    import spark.implicits._
 
     spark.read.load(parser.get("sourcePath")).as[CrossrefDT]
       .flatMap(k => Crossref2Oaf.convert(k.json))

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/mag/MagDataModel.scala

@@ -188,7 +188,7 @@ case object ConversionUtil {
     val authors = inputParams._2
 
     val pub = new Publication
-    pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava)
+    pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
     pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava)
 
     //Set identifier as 50|doiboost____::md5(DOI)
@@ -247,7 +247,7 @@ case object ConversionUtil {
     val description = inputParams._2
 
     val pub = new Publication
-    pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", PID_TYPES)).asJava)
+    pub.setPid(List(createSP(paper.Doi.toLowerCase, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
     pub.setOriginalId(List(paper.PaperId.toString, paper.Doi.toLowerCase).asJava)
 
     //Set identifier as 50 | doiboost____::md5(DOI)

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/orcidnodoi/oaf/PublicationToOaf.java

@@ -30,7 +30,6 @@ public class PublicationToOaf implements Serializable {
 
 	static Logger logger = LoggerFactory.getLogger(PublicationToOaf.class);
 
-	public static final String ORCID = "ORCID";
 	public final static String orcidPREFIX = "orcid_______";
 	public static final String OPENAIRE_PREFIX = "openaire____";
 	public static final String SEPARATOR = "::";
@@ -69,7 +68,9 @@ public class PublicationToOaf implements Serializable {
 	private static Map<String, Pair<String, String>> datasources = new HashMap<String, Pair<String, String>>() {
 
 		{
-			put(ORCID.toLowerCase(), new Pair<>(ORCID, OPENAIRE_PREFIX + SEPARATOR + "orcid"));
+			put(
+				ModelConstants.ORCID,
+				new Pair<>(ModelConstants.ORCID_DS, OPENAIRE_PREFIX + SEPARATOR + ModelConstants.ORCID));
 
 		}
 	};
@@ -102,8 +103,6 @@ public class PublicationToOaf implements Serializable {
 		}
 	}
 
-	public static final String PID_TYPES = "dnet:pid_types";
-
 	public Oaf generatePublicationActionsFromJson(final String json) {
 		try {
 			if (parsedPublications != null) {
@@ -138,8 +137,8 @@ public class PublicationToOaf implements Serializable {
 				mapQualifier(
 					"sysimport:actionset:orcidworks-no-doi",
 					"sysimport:actionset:orcidworks-no-doi",
-					"dnet:provenanceActions",
+					ModelConstants.DNET_PROVENANCE_ACTIONS,
-					"dnet:provenanceActions"));
+					ModelConstants.DNET_PROVENANCE_ACTIONS));
 		publication.setDataInfo(dataInfo);
 
 		publication.setLastupdatetimestamp(new Date().getTime());
@@ -159,7 +158,9 @@ public class PublicationToOaf implements Serializable {
 					publication
 						.getExternalReference()
 						.add(
-							convertExtRef(extId, classid, classname, "dnet:pid_types", "dnet:pid_types"));
+							convertExtRef(
+								extId, classid, classname, ModelConstants.DNET_PID_TYPES,
+								ModelConstants.DNET_PID_TYPES));
 				}
 			});
 
@@ -505,24 +506,21 @@ public class PublicationToOaf implements Serializable {
 
 	private KeyValue createCollectedFrom() {
 		KeyValue cf = new KeyValue();
-		cf.setValue(ORCID);
+		cf.setValue(ModelConstants.ORCID_DS);
 		cf.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "806360c771262b4d6770e7cdf04b5c5a");
 		return cf;
 	}
 
 	private KeyValue createHostedBy() {
-		KeyValue hb = new KeyValue();
+		return ModelConstants.UNKNOWN_REPOSITORY;
-		hb.setValue("Unknown Repository");
-		hb.setKey("10|" + OPENAIRE_PREFIX + SEPARATOR + "55045bd2a65019fd8e6741a755395c8c");
-		return hb;
 	}
 
 	private StructuredProperty mapAuthorId(String orcidId) {
 		final StructuredProperty sp = new StructuredProperty();
 		sp.setValue(orcidId);
 		final Qualifier q = new Qualifier();
-		q.setClassid(ORCID.toLowerCase());
+		q.setClassid(ModelConstants.ORCID);
-		q.setClassname(ORCID.toLowerCase());
+		q.setClassname(ModelConstants.ORCID_CLASSNAME);
 		q.setSchemeid(ModelConstants.DNET_PID_TYPES);
 		q.setSchemename(ModelConstants.DNET_PID_TYPES);
 		sp.setQualifier(q);
@@ -535,8 +533,8 @@ public class PublicationToOaf implements Serializable {
 				mapQualifier(
 					"sysimport:crosswalk:entityregistry",
 					"Harvested",
-					"dnet:provenanceActions",
+					ModelConstants.DNET_PROVENANCE_ACTIONS,
-					"dnet:provenanceActions"));
+					ModelConstants.DNET_PROVENANCE_ACTIONS));
 		sp.setDataInfo(dataInfo);
 		return sp;
 	}

dhp-workflows/dhp-doiboost/src/main/java/eu/dnetlib/doiboost/uw/UnpayWallToOAF.scala

@@ -1,5 +1,6 @@
 package eu.dnetlib.doiboost.uw
 
+import eu.dnetlib.dhp.schema.common.ModelConstants
 import eu.dnetlib.dhp.schema.oaf.{Instance, Publication}
 import org.json4s
 import org.json4s.DefaultFormats
@@ -32,7 +33,7 @@ object UnpayWallToOAF {
     val is_oa = (json\ "is_oa").extract[Boolean]
 
     val oaLocation:OALocation = (json \ "best_oa_location").extractOrElse[OALocation](null)
-    pub.setPid(List(createSP(doi, "doi", PID_TYPES)).asJava)
+    pub.setPid(List(createSP(doi, "doi", ModelConstants.DNET_PID_TYPES)).asJava)
     pub.setId(generateIdentifier(pub, doi.toLowerCase))
 
     pub.setCollectedfrom(List(createUnpayWallCollectedFrom()).asJava)

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/config-default.xml

@@ -1,42 +0,0 @@
-<configuration>
-    <property>
-        <name>jobTracker</name>
-        <value>yarnRM</value>
-    </property>
-    <property>
-        <name>nameNode</name>
-        <value>hdfs://nameservice1</value>
-    </property>
-    <property>
-        <name>oozie.use.system.libpath</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>oozie.action.sharelib.for.spark</name>
-        <value>spark2</value>
-    </property>
-    <property>
-        <name>oozie.launcher.mapreduce.user.classpath.first</name>
-        <value>true</value>
-    </property>
-    <property>
-        <name>hive_metastore_uris</name>
-        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
-    </property>
-    <property>
-        <name>spark2YarnHistoryServerAddress</name>
-        <value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089</value>
-    </property>
-    <property>
-        <name>spark2EventLogDir</name>
-        <value>/user/spark/spark2ApplicationHistory</value>
-    </property>
-    <property>
-        <name>spark2ExtraListeners</name>
-        <value>"com.cloudera.spark.lineage.NavigatorAppListener"</value>
-    </property>
-    <property>
-        <name>spark2SqlQueryExecutionListeners</name>
-        <value>"com.cloudera.spark.lineage.NavigatorQueryListener"</value>
-    </property>
-</configuration>

dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/oozie_app/workflow.xml

@@ -1,372 +0,0 @@
-<workflow-app name="Generate DOIBoost ActionSet" xmlns="uri:oozie:workflow:0.5">
-    <parameters>
-        <property>
-            <name>sparkDriverMemory</name>
-            <description>memory for driver process</description>
-        </property>
-        <property>
-            <name>sparkExecutorMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-        <property>
-            <name>sparkExecutorIntersectionMemory</name>
-            <description>memory for individual executor</description>
-        </property>
-
-        <property>
-            <name>sparkExecutorCores</name>
-            <description>number of cores used by single executor</description>
-        </property>
-
-
-        <!-- Itersection Parameters -->
-        <property>
-            <name>workingPath</name>
-            <description>the working Path</description>
-        </property>
-
-        <property>
-            <name>hostedByMapPath</name>
-            <description>the hostedByMap Path</description>
-        </property>
-        <property>
-            <name>outputPath</name>
-            <description>the Path of the sequence file action set</description>
-        </property>
-
-
-        <!-- Crossref Parameters -->
-        <property>
-            <name>inputPathCrossref</name>
-            <description>the Crossref input path</description>
-        </property>
-        <property>
-            <name>crossrefTimestamp</name>
-            <description>Timestamp for the Crossref incremental Harvesting</description>
-        </property>
-        <property>
-            <name>esServer</name>
-            <description>elasticsearch server url for the Crossref Harvesting</description>
-        </property>
-        <property>
-            <name>esIndex</name>
-            <description>elasticsearch index name for the Crossref Harvesting</description>
-        </property>
-
-        <!--    MAG Parameters    -->
-        <property>
-            <name>MAGDumpPath</name>
-            <description>the MAG dump working path</description>
-        </property>
-
-        <property>
-            <name>inputPathMAG</name>
-            <description>the MAG working path</description>
-        </property>
-
-
-        <!--    UnpayWall Parameters    -->
-        <property>
-            <name>inputPathUnpayWall</name>
-            <description>the UnpayWall working path</description>
-        </property>
-
-        <!--    ORCID Parameters    -->
-        <property>
-            <name>inputPathOrcid</name>
-            <description>the ORCID input path</description>
-        </property>
-
-        <property>
-            <name>workingPathOrcid</name>
-            <description>the ORCID working path</description>
-        </property>
-
-    </parameters>
-
-    <global>
-        <job-tracker>${jobTracker}</job-tracker>
-        <name-node>${nameNode}</name-node>
-        <configuration>
-            <property>
-                <name>oozie.action.sharelib.for.spark</name>
-                <value>${oozieActionShareLibForSpark2}</value>
-            </property>
-        </configuration>
-    </global>
-
-    <start to="resume_from"/>
-
-    <decision name="resume_from">
-        <switch>
-            <case to="ConvertCrossrefToOAF">${wf:conf('resumeFrom') eq 'ConvertCrossrefToOAF'}</case>
-            <case to="ResetMagWorkingPath">${wf:conf('resumeFrom') eq 'ResetMagWorkingPath'}</case>
-            <case to="ProcessMAG">${wf:conf('resumeFrom') eq 'PreprocessMag'}</case>
-            <case to="ProcessUW">${wf:conf('resumeFrom') eq 'PreprocessUW'}</case>
-            <case to="ProcessORCID">${wf:conf('resumeFrom') eq 'PreprocessORCID'}</case>
-            <case to="CreateDOIBoost">${wf:conf('resumeFrom') eq 'CreateDOIBoost'}</case>
-            <case to="GenerateActionSet">${wf:conf('resumeFrom') eq 'GenerateActionSet'}</case>
-            <default to="ImportCrossRef"/>
-        </switch>
-    </decision>
-
-    <kill name="Kill">
-        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
-    </kill>
-
-    <action name="ImportCrossRef">
-        <java>
-            <main-class>eu.dnetlib.doiboost.crossref.CrossrefImporter</main-class>
-            <arg>--targetPath</arg><arg>${inputPathCrossref}/index_update</arg>
-            <arg>--namenode</arg><arg>${nameNode}</arg>
-            <arg>--esServer</arg><arg>${esServer}</arg>
-            <arg>--esIndex</arg><arg>${esIndex}</arg>
-            <arg>--timestamp</arg><arg>${crossrefTimestamp}</arg>
-        </java>
-        <ok to="GenerateCrossrefDataset"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <!-- CROSSREF SECTION -->
-
-    <action name="GenerateCrossrefDataset">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>GenerateCrossrefDataset</name>
-            <class>eu.dnetlib.doiboost.crossref.CrossrefDataset</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--workingPath</arg><arg>${inputPathCrossref}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="RenameDataset"/>
-        <error to="Kill"/>
-    </action>
-
-    <action name="RenameDataset">
-        <fs>
-            <delete path="${inputPathCrossref}/crossref_ds"/>
-            <move source="${inputPathCrossref}/crossref_ds_updated"
-                  target="${inputPathCrossref}/crossref_ds"/>
-        </fs>
-        <ok to="ResetMagWorkingPath"/>
-        <error to="Kill"/>
-    </action>
-
-
-
-    <!-- MAG SECTION -->
-    <action name="ResetMagWorkingPath">
-        <fs>
-            <delete path="${inputPathMAG}/dataset"/>
-            <delete path="${inputPathMAG}/process"/>
-        </fs>
-        <ok to="ConvertMagToDataset"/>
-        <error to="Kill"/>
-    </action>
-
-    <action name="ConvertMagToDataset">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Convert Mag to Dataset</name>
-            <class>eu.dnetlib.doiboost.mag.SparkImportMagIntoDataset</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${MAGDumpPath}</arg>
-            <arg>--targetPath</arg><arg>${inputPathMAG}/dataset</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="ConvertCrossrefToOAF"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="ConvertCrossrefToOAF">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>ConvertCrossrefToOAF</name>
-            <class>eu.dnetlib.doiboost.crossref.SparkMapDumpIntoOAF</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${inputPathCrossref}/crossref_ds</arg>
-            <arg>--targetPath</arg><arg>${workingPath}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="ProcessMAG"/>
-        <error to="Kill"/>
-    </action>
-
-    <action name="ProcessMAG">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Convert Mag to OAF Dataset</name>
-            <class>eu.dnetlib.doiboost.mag.SparkProcessMAG</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorIntersectionMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${inputPathMAG}/dataset</arg>
-            <arg>--workingPath</arg><arg>${inputPathMAG}/process</arg>
-            <arg>--targetPath</arg><arg>${workingPath}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="ProcessUW"/>
-        <error to="Kill"/>
-    </action>
-
-    <!--  UnpayWall  SECTION -->
-
-    <action name="ProcessUW">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Convert UnpayWall to Dataset</name>
-            <class>eu.dnetlib.doiboost.uw.SparkMapUnpayWallToOAF</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${inputPathUnpayWall}/uw_extracted</arg>
-            <arg>--targetPath</arg><arg>${workingPath}/uwPublication</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="ProcessORCID"/>
-        <error to="Kill"/>
-    </action>
-
-    <!--  ORCID  SECTION -->
-    <action name="ProcessORCID">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Convert ORCID to Dataset</name>
-            <class>eu.dnetlib.doiboost.orcid.SparkConvertORCIDToOAF</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--sourcePath</arg><arg>${inputPathOrcid}</arg>
-            <arg>--workingPath</arg><arg>${workingPathOrcid}</arg>
-            <arg>--targetPath</arg><arg>${workingPath}/orcidPublication</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="CreateDOIBoost"/>
-        <error to="Kill"/>
-    </action>
-
-    <!-- INTERSECTION SECTION-->
-    <action name="CreateDOIBoost">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Create DOIBoost Infospace</name>
-            <class>eu.dnetlib.doiboost.SparkGenerateDoiBoost</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorIntersectionMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--hostedByMapPath</arg><arg>${hostedByMapPath}</arg>
-            <arg>--affiliationPath</arg><arg>${inputPathMAG}/dataset/Affiliations</arg>
-            <arg>--paperAffiliationPath</arg><arg>${inputPathMAG}/dataset/PaperAuthorAffiliations</arg>
-            <arg>--workingPath</arg><arg>${workingPath}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="GenerateActionSet"/>
-        <error to="Kill"/>
-    </action>
-
-
-    <action name="GenerateActionSet">
-        <spark xmlns="uri:oozie:spark-action:0.2">
-            <master>yarn-cluster</master>
-            <mode>cluster</mode>
-            <name>Generate DOIBoost ActionSet</name>
-            <class>eu.dnetlib.doiboost.SparkGenerateDOIBoostActionSet</class>
-            <jar>dhp-doiboost-${projectVersion}.jar</jar>
-            <spark-opts>
-                --executor-memory=${sparkExecutorMemory}
-                --executor-cores=${sparkExecutorCores}
-                --driver-memory=${sparkDriverMemory}
-                --conf spark.sql.shuffle.partitions=3840
-                --conf spark.extraListeners=${spark2ExtraListeners}
-                --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
-                --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
-                --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
-            </spark-opts>
-            <arg>--dbPublicationPath</arg><arg>${workingPath}/doiBoostPublicationFiltered</arg>
-            <arg>--dbDatasetPath</arg><arg>${workingPath}/crossrefDataset</arg>
-            <arg>--crossRefRelation</arg><arg>${workingPath}/crossrefRelation</arg>
-            <arg>--dbaffiliationRelationPath</arg><arg>${workingPath}/doiBoostPublicationAffiliation</arg>
-            <arg>--dbOrganizationPath</arg><arg>${workingPath}/doiBoostOrganization</arg>
-            <arg>--targetPath</arg><arg>${workingPath}/actionDataSet</arg>
-            <arg>--sFilePath</arg><arg>${outputPath}</arg>
-            <arg>--master</arg><arg>yarn-cluster</arg>
-        </spark>
-        <ok to="End"/>
-        <error to="Kill"/>
-    </action>
-
-    <end name="End"/>
-</workflow-app>

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/Constants.java

@@ -24,8 +24,6 @@ public class Constants {
 
 	public static String RESEARCH_INFRASTRUCTURE = "Research Infrastructure/Initiative";
 
-	public static String ORCID = "orcid";
-
 	static {
 		accessRightsCoarMap.put("OPEN", "c_abf2");
 		accessRightsCoarMap.put("RESTRICTED", "c_16ec");

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/ResultMapper.java

@@ -503,7 +503,7 @@ public class ResultMapper implements Serializable {
 
 	private static Pid getOrcid(List<StructuredProperty> p) {
 		for (StructuredProperty pid : p) {
-			if (pid.getQualifier().getClassid().equals(Constants.ORCID)) {
+			if (pid.getQualifier().getClassid().equals(ModelConstants.ORCID)) {
 				Optional<DataInfo> di = Optional.ofNullable(pid.getDataInfo());
 				if (di.isPresent()) {
 					return Pid

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java

@@ -68,7 +68,7 @@ public abstract class AbstractMdRecordToOafMapper {
 	protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3";
 	protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/";
 	protected static final Qualifier ORCID_PID_TYPE = qualifier(
-		"ORCID", "Open Researcher and Contributor ID", DNET_PID_TYPES, DNET_PID_TYPES);
+		ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, DNET_PID_TYPES, DNET_PID_TYPES);
 	protected static final Qualifier MAG_PID_TYPE = qualifier(
 		"MAGIdentifier", "Microsoft Academic Graph Identifier", DNET_PID_TYPES, DNET_PID_TYPES);
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java

@@ -19,6 +19,7 @@ import com.google.common.collect.Lists;
 
 import eu.dnetlib.dhp.common.PacePerson;
 import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.Author;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Field;
@@ -61,7 +62,7 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper {
 			author.setPid(new ArrayList<>());
 
 			if (StringUtils.isNotBlank(pid)) {
-				if (type.startsWith("ORCID")) {
+				if (type.toLowerCase().startsWith(ORCID)) {
 					final String cleanedId = pid
 						.replaceAll("http://orcid.org/", "")
 						.replaceAll("https://orcid.org/", "");

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java

@@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
 
 import eu.dnetlib.dhp.common.PacePerson;
 import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
+import eu.dnetlib.dhp.schema.common.ModelConstants;
 import eu.dnetlib.dhp.schema.oaf.Author;
 import eu.dnetlib.dhp.schema.oaf.DataInfo;
 import eu.dnetlib.dhp.schema.oaf.Field;
@@ -98,7 +99,7 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper {
 				.replaceAll(" ", "")
 				.replaceAll("_", "");
 
-			if (type.startsWith("ORCID")) {
+			if (type.toLowerCase().startsWith(ModelConstants.ORCID)) {
 				final String cleanedId = id.replaceAll("http://orcid.org/", "").replaceAll("https://orcid.org/", "");
 				res.add(structuredProperty(cleanedId, ORCID_PID_TYPE, info));
 			} else if (type.startsWith("MAGID")) {

dhp-workflows/dhp-indicators/nb-configuration.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project-shared-configuration>
+    <!--
+This file contains additional configuration written by modules in the NetBeans IDE.
+The configuration is intended to be shared among all the users of project and
+therefore it is assumed to be part of version control checkout.
+Without this configuration present, some functionality in the IDE may be limited or fail altogether.
+-->
+    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
+        <!--
+Properties that influence various parts of the IDE, especially code formatting and the like. 
+You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
+That way multiple projects can share the same settings (useful for formatting rules for example).
+Any value defined here will override the pom.xml file value but is only applicable to the current project.
+-->
+        <netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
+    </properties>
+</project-shared-configuration>

dhp-workflows/dhp-indicators/pom.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+
+    <parent>
+        <artifactId>dhp-workflows</artifactId>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <version>1.1.7-SNAPSHOT</version>
+        <relativePath>../</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>dhp-indicators</artifactId>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>2.1.15</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
+                    <!-- more config here as you see fit -->
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.6.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins> 
+    </build>  
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <cdh.hive.version>0.13.1-cdh5.2.1</cdh.hive.version>
+        <cdh.hadoop.version>2.5.0-cdh5.2.1</cdh.hadoop.version>
+	</properties>
+    
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_2.11</artifactId>
+            <version>2.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_2.11</artifactId>
+            <version>2.4.5</version>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.json-simple</groupId>
+            <artifactId>json-simple</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+            <version>20180130</version>
+            <type>jar</type>
+        </dependency>
+		<dependency>
+			<groupId>org.apache.hive</groupId>
+			<artifactId>hive-jdbc</artifactId>
+			<version>${cdh.hive.version}</version>
+		</dependency>
+		<dependency>
+  			<groupId>org.apache.hadoop</groupId>
+  			<artifactId>hadoop-common</artifactId>
+  			<version>2.7.4</version>
+		 <type>jar</type>
+		</dependency>        
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-common</artifactId>
+            <version>1.1.7-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>com.mchange</groupId>
+            <artifactId>c3p0</artifactId>
+            <version>0.9.5.2</version>
+        </dependency>
+        <dependency>
+            <groupId>c3p0</groupId>
+            <artifactId>c3p0</artifactId>
+            <version>0.9.1.2</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.26</version>
+            <type>jar</type>
+        </dependency>
+    </dependencies>
+    <name>dhp-indicators</name>
+</project>

dhp-workflows/dhp-indicators/runworkflow.sh

@@ -0,0 +1 @@
+mvn clean package -Poozie-package,deploy,run -Dworkflow.source.dir=eu/dnetlib/dhp/oa/graph/indicators

dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/config-default.xml

@@ -0,0 +1,34 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>${jobTracker}</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>${nameNode}</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+    <property>
+        <name>hive_metastore_uris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hive_jdbc_url</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
+    </property>
+	<property>
+		<name>oozie.wf.workflow.notification.url</name>
+		<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
+	</property>
+<!--    <property>
+        <name>stats_tool_api_url</name>
+        <value>${stats_tool_api_url}</value>
+    </property>-->
+</configuration>

dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/createIndicatorsTables.sql

@@ -0,0 +1,47 @@
+create table TARGET.indi_pub_green_oa stored as parquet as
+select distinct p.id, coalesce(green_oa, 0) as green_oa 
+from SOURCE.publication p 
+left outer join ( 
+select p.id, 1 as green_oa 
+from SOURCE.publication p 
+join SOURCE.result_instance ri on ri.id = p.id 
+join SOURCE.datasource on datasource.id = ri.hostedby  
+where SOURCE.datasource.type like '%Repository%'  
+and (ri.accessright = 'Open Access'  
+or ri.accessright = 'Embargo')) tmp 
+on p.id= tmp.id;
+
+create table TARGET.indi_pub_grey_lit stored as parquet as
+select distinct p.id, coalesce(grey_lit, 0) as grey_lit
+from SOURCE.publication p
+left outer join (
+select p.id, 1 as grey_lit 
+from SOURCE.publication p
+join SOURCE.result_classifications rt on rt.id = p.id
+where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and 
+not exists (select 1 from SOURCE.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id;
+
+create table TARGET.indi_pub_doi_from_crossref stored as parquet as
+select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref 
+from SOURCE.publication p
+left outer join
+(select ri.id, 1 as doi_from_crossref from SOURCE.result_instance ri
+join SOURCE.datasource d on d.id = ri.collectedfrom
+where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp
+on tmp.id=p.id;
+
+create table TARGET.indi_pub_gold_oa stored as parquet as
+select distinct p.id, coalesce(gold_oa, 0) as gold_oa
+from SOURCE.publication p 
+left outer join ( 
+select p.id, 1 as gold_oa
+from SOURCE.publication p 
+join SOURCE.result_instance ri on ri.id = p.id 
+join SOURCE.datasource on datasource.id = ri.hostedby  
+where SOURCE.datasource.id like '%doajarticles%') tmp 
+on p.id= tmp.id;
+
+compute stats TARGET.indi_pub_green_oa;
+compute stats TARGET.indi_pub_grey_lit;
+compute stats TARGET.indi_pub_doi_from_crossref;
+compute stats TARGET.indi_pub_gold_oa;

dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/scripts/indicators.sh

@@ -0,0 +1,29 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+export SOURCE=$1
+export TARGET=$2
+export SHADOW=$3
+export SCRIPT_PATH=$4
+
+echo "Getting file from " $4
+hdfs dfs -copyToLocal $4
+
+echo "Creating indicators database"
+impala-shell -q "drop database if exists ${TARGET} cascade"
+impala-shell -q "create database if not exists ${TARGET}"
+impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${TARGET}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -f -
+cat createIndicatorsTables.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f -
+echo "Indicators Database created"
+
+
+echo "Updating Shadow indicators DB"
+impala-shell -q "create database if not exists ${SHADOW}"
+impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
+impala-shell -d ${TARGET} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${TARGET}.\1;/" | impala-shell -f -
+echo "Indicators Shadow DB ready!"

dhp-workflows/dhp-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/indicators/oozie_app/workflow.xml

@@ -0,0 +1,101 @@
+<workflow-app name="Indicators" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>stats_db_name</name>
+            <description>the source stats database name</description>
+        </property>
+        <property>
+            <name>indicators_db_name</name>
+            <description>the target indicators database name</description>
+        </property>
+        <property>
+            <name>indicators_shadow_db_name</name>
+            <description>the name of the shadow schema</description>
+        </property>
+        <!--        <property>
+            <name>openaire_db_name</name>
+            <description>the original graph database name</description>
+        </property>
+        <property>
+            <name>external_stats_db_name</name>
+            <value>stats_ext</value>
+            <description>the external stats that should be added since they are not included in the graph database</description>
+        </property>
+        <property>
+            <name>stats_db_shadow_name</name>
+            <description>the name of the shadow schema</description>
+        </property>
+        <property>
+            <name>monitor_db_name</name>
+            <description>the target monitor db name</description>
+        </property>
+        <property>
+            <name>monitor_db_shadow_name</name>
+            <description>the name of the shadow monitor db</description>
+        </property>
+        <property>
+            <name>observatory_db_name</name>
+            <description>the target monitor db name</description>
+        </property>
+        <property>
+            <name>observatory_db_shadow_name</name>
+            <description>the name of the shadow monitor db</description>
+        </property>
+        <property>
+            <name>stats_tool_api_url</name>
+            <description>The url of the API of the stats tool. Is used to trigger the cache update.</description>
+        </property>-->
+        <property>
+            <name>hive_metastore_uris</name>
+            <description>hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>hive_jdbc_url</name>
+            <description>hive server jdbc url</description>
+        </property>
+        <!--        <property>
+            <name>hive_timeout</name>
+            <description>the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds.</description>
+        </property>-->
+        <!--        <property>
+            <name>context_api_url</name>
+            <description>the base url of the context api (https://services.openaire.eu/openaire)</description>
+        </property>-->
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>hive.metastore.uris</name>
+                <value>${hive_metastore_uris}</value>
+            </property>
+            <!--            <property>
+                <name>hive.txn.timeout</name>
+                <value>${hive_timeout}</value>
+            </property>-->
+        </configuration>
+    </global>
+
+    <start to="Step1-createIndicatorsDB"/>
+    <action name="Step1-createIndicatorsDB">
+        <shell xmlns="uri:oozie:shell-action:0.1">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <exec>indicators.sh</exec>
+            <argument>${stats_db_name}</argument>
+            <argument>${indicators_db_name}</argument>
+            <argument>${indicators_shadow_db_name}</argument>
+            <argument>${wf:appPath()}/scripts/createIndicatorsTables.sql</argument>
+            <file>scripts/indicators.sh</file>
+        </shell>
+        <ok to="End"/>
+        <error to="Kill"/>
+    </action>
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml

@@ -16,6 +16,14 @@
             <name>monitor_db_production_name</name>
             <description>the name of the monitor public database</description>
         </property>
+        <property>
+            <name>observatory_db_name</name>
+            <description>the monitor database name</description>
+        </property>
+        <property>
+            <name>observatory_db_production_name</name>
+            <description>the name of the monitor public database</description>
+        </property>
         <property>
             <name>stats_tool_api_url</name>
             <description>The url of the API of the stats tool. Is used to trigger the cache promote.</description>
@@ -77,6 +85,19 @@
             <argument>${monitor_db_production_name}</argument>
             <file>updateProductionViews.sh</file>
         </shell>
+        <ok to="updateObservatoryViews"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="updateObservatoryViews">
+        <shell xmlns="uri:oozie:shell-action:0.1">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <exec>updateProductionViews.sh</exec>
+            <argument>${observatory_db_name}</argument>
+            <argument>${observatory_db_production_name}</argument>
+            <file>updateProductionViews.sh</file>
+        </shell>
         <ok to="promoteCache"/>
         <error to="Kill"/>
     </action>

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/observatory.sh

@@ -0,0 +1,28 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+export SOURCE=$1
+export TARGET=$2
+export SHADOW=$3
+export SCRIPT_PATH=$4
+
+echo "Getting file from " $4
+hdfs dfs -copyToLocal $4
+
+echo "Creating observatory database"
+impala-shell -q "drop database if exists ${TARGET} cascade"
+impala-shell -q "create database if not exists ${TARGET}"
+impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${TARGET}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -f -
+cat step21-createObservatoryDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f -
+echo "Impala shell finished"
+
+echo "Updating shadow observatory database"
+impala-shell -q "create database if not exists ${SHADOW}"
+impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
+impala-shell -d ${TARGET} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${TARGET}.\1;/" | impala-shell -f -
+echo "Shadow db ready!"

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql

@@ -45,35 +45,3 @@ FROM ${stats_db_name}.dataset
 UNION ALL
 SELECT *, bestlicence AS access_mode
 FROM ${stats_db_name}.otherresearchproduct;
-
-
--------------------------------------------------------------------------------
--- To see with Antonis if the following is needed and where it should be placed
--------------------------------------------------------------------------------
-CREATE TABLE ${stats_db_name}.numbers_country AS
-SELECT org.country AS country, count(distinct rd.datasource) AS datasources, count(distinct r.id) AS publications
-FROM ${stats_db_name}.result r,
-     ${stats_db_name}.result_datasources rd,
-     ${stats_db_name}.datasource d,
-     ${stats_db_name}.datasource_organizations dor,
-     ${stats_db_name}.organization org
-WHERE r.id = rd.id
-  AND rd.datasource = d.id
-  AND d.id = dor.id
-  AND dor.organization = org.id
-  AND r.type = 'publication'
-  AND r.bestlicence = 'Open Access'
-GROUP BY org.country;
-
--- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16.sql

@@ -60,32 +60,3 @@ union all
 select distinct r.id, false as gold
 from ${stats_db_name}.result r
 where r.id not in (select id from result_gold);
-
--- shortcut result-country through the organization affiliation
-create table ${stats_db_name}.result_affiliated_country as
-select r.id as id, o.country as country
-from ${stats_db_name}.result r
-join ${stats_db_name}.result_organization ro on ro.id=r.id
-join ${stats_db_name}.organization o on o.id=ro.organization
-where o.country is not null and o.country!='';
-
--- shortcut result-country through datasource of deposition
-create table ${stats_db_name}.result_deposited_country as
-select r.id as id, o.country as country
-from ${stats_db_name}.result r
-join ${stats_db_name}.result_datasources rd on rd.id=r.id
-join ${stats_db_name}.datasource d on d.id=rd.datasource
-join ${stats_db_name}.datasource_organizations dor on dor.id=d.id
-join ${stats_db_name}.organization o on o.id=dor.organization
-where o.country is not null and o.country!='';
-
--- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS FOR COLUMNS;
--- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql

@@ -53,6 +53,3 @@ drop table if exists ${stats_db_name}.result;
 drop view if exists ${stats_db_name}.result;
 create table ${stats_db_name}.result stored as parquet as select * from ${stats_db_name}.result_tmp;
 drop table ${stats_db_name}.result_tmp;
---
--- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql

@@ -19,9 +19,6 @@ create table TARGET.result as
         select * from SOURCE.result r where exists (select 1 from SOURCE.result_concepts rc where rc.id=r.id) ) foo;
 compute stats TARGET.result;
 
-create table TARGET.result_affiliated_country as select * from SOURCE.result_affiliated_country rac where exists (select 1 from TARGET.result r where r.id=rac.id);
-compute stats TARGET.result_affiliated_country;
-
 create table TARGET.result_citations as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_citations;
 
@@ -34,9 +31,6 @@ compute stats TARGET.result_concepts;
 create table TARGET.result_datasources as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_datasources;
 
-create table TARGET.result_deposited_country as select * from SOURCE.result_deposited_country orig where exists (select 1 from TARGET.result r where r.id=orig.id);
-compute stats TARGET.result_deposited_country;
-
 create table TARGET.result_fundercount as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_fundercount;
 

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql

@@ -0,0 +1,259 @@
+create table TARGET.result_affiliated_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, c.code, c.name;
+
+create table TARGET.result_affiliated_year stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, r.year
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year;
+
+create table TARGET.result_affiliated_year_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, r.year, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year, c.code, c.name;
+
+create table TARGET.result_affiliated_datasource stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, d.name as dname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_datasources rd on rd.id=r.id
+left outer join SOURCE.datasource d on d.id=rd.datasource
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name;
+
+create table TARGET.result_affiliated_datasource_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, d.name as dname, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_datasources rd on rd.id=r.id
+left outer join SOURCE.datasource d on d.id=rd.datasource
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name, c.code, c.name;
+
+create table TARGET.result_affiliated_organization stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, o.name as oname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name;
+
+create table TARGET.result_affiliated_organization_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, o.name as oname, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name, c.code, c.name;
+
+create table TARGET.result_affiliated_funder stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, p.funder as pfunder
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+join SOURCE.result_projects rp on rp.id=r.id
+join SOURCE.project p on p.id=rp.project
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder;
+
+create table TARGET.result_affiliated_funder_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, p.funder as pfunder, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_organization ro on ro.id=r.id
+join SOURCE.organization o on o.id=ro.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+join SOURCE.result_projects rp on rp.id=r.id
+join SOURCE.project p on p.id=rp.project
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder, c.code, c.name;
+
+create table TARGET.result_deposited_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, c.code, c.name;
+
+create table TARGET.result_deposited_year stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, r.year
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year;
+
+create table TARGET.result_deposited_year_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, r.year, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, r.year, c.code, c.name;
+
+create table TARGET.result_deposited_datasource stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, d.name as dname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name;
+
+create table TARGET.result_deposited_datasource_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, d.name as dname, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, d.name, c.code, c.name;
+
+create table TARGET.result_deposited_organization stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa, r.peer_reviewed, r.type, o.name as oname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name;
+
+create table TARGET.result_deposited_organization_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, o.name as oname, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, o.name, c.code, c.name;
+
+create table TARGET.result_deposited_funder stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, p.funder as pfunder
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+join SOURCE.result_projects rp on rp.id=r.id
+join SOURCE.project p on p.id=rp.project
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder;
+
+create table TARGET.result_deposited_funder_country stored as parquet as
+select count(distinct r.id) as total, r.green, r.gold, case when rl.type is not null then true else false end as licence,
+  case when pids.pid is not null then true else false end as pid, case when r.access_mode in ('Open Access', 'Open Source') then true else false end as oa,
+  r.peer_reviewed, r.type, p.funder as pfunder, c.code as ccode, c.name as cname
+from SOURCE.result r
+join SOURCE.result_datasources rd on rd.id=r.id
+join SOURCE.datasource d on d.id=rd.datasource and d.type in ('Institutional Repository','Data Repository', 'Repository', 'Publication Repository')
+join SOURCE.datasource_organizations dor on dor.id=d.id
+join SOURCE.organization o on o.id=dor.organization
+join SOURCE.country c on c.code=o.country and c.continent_name='Europe'
+join SOURCE.result_projects rp on rp.id=r.id
+join SOURCE.project p on p.id=rp.project
+left outer join SOURCE.result_licenses rl on rl.id=r.id
+left outer join SOURCE.result_pids pids on pids.id=r.id
+group by r.green, r.gold, licence, pid, oa, r.peer_reviewed, r.type, p.funder, c.code, c.name;
+
+compute stats TARGET.result_affiliated_country;
+compute stats TARGET.result_affiliated_year;
+compute stats TARGET.result_affiliated_year_country;
+compute stats TARGET.result_affiliated_datasource;
+compute stats TARGET.result_affiliated_datasource_country;
+compute stats TARGET.result_affiliated_organization;
+compute stats TARGET.result_affiliated_organization_country;
+compute stats TARGET.result_affiliated_funder;
+compute stats TARGET.result_affiliated_funder_country;
+compute stats TARGET.result_deposited_country;
+compute stats TARGET.result_deposited_year;
+compute stats TARGET.result_deposited_year_country;
+compute stats TARGET.result_deposited_datasource;
+compute stats TARGET.result_deposited_datasource_country;
+compute stats TARGET.result_deposited_organization;
+compute stats TARGET.result_deposited_organization_country;
+compute stats TARGET.result_deposited_funder;
+compute stats TARGET.result_deposited_funder_country;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml

@@ -25,6 +25,14 @@
             <name>monitor_db_shadow_name</name>
             <description>the name of the shadow monitor db</description>
         </property>
+        <property>
+            <name>observatory_db_name</name>
+            <description>the target monitor db name</description>
+        </property>
+        <property>
+            <name>observatory_db_shadow_name</name>
+            <description>the name of the shadow monitor db</description>
+        </property>
         <property>
             <name>stats_tool_api_url</name>
             <description>The url of the API of the stats tool. Is used to trigger the cache update.</description>
@@ -305,11 +313,26 @@
             <argument>${wf:appPath()}/scripts/step20-createMonitorDB.sql</argument>
             <file>monitor.sh</file>
         </shell>
-        <ok to="Step21"/>
+        <ok to="step21-createObservatoryDB"/>
         <error to="Kill"/>
     </action>
 
-    <action name="Step21">
+    <action name="step21-createObservatoryDB">
+        <shell xmlns="uri:oozie:shell-action:0.1">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <exec>observatory.sh</exec>
+            <argument>${stats_db_name}</argument>
+            <argument>${observatory_db_name}</argument>
+            <argument>${observatory_db_shadow_name}</argument>
+            <argument>${wf:appPath()}/scripts/step21-createObservatoryDB.sql</argument>
+            <file>observatory.sh</file>
+        </shell>
+        <ok to="Step22"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="Step22">
         <shell xmlns="uri:oozie:shell-action:0.1">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>

dhp-workflows/dhp-usage-datasets-stats-update/nb-configuration.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project-shared-configuration>
+    <!--
+This file contains additional configuration written by modules in the NetBeans IDE.
+The configuration is intended to be shared among all the users of project and
+therefore it is assumed to be part of version control checkout.
+Without this configuration present, some functionality in the IDE may be limited or fail altogether.
+-->
+    <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1">
+        <!--
+Properties that influence various parts of the IDE, especially code formatting and the like. 
+You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up.
+That way multiple projects can share the same settings (useful for formatting rules for example).
+Any value defined here will override the pom.xml file value but is only applicable to the current project.
+-->
+        <netbeans.hint.jdkPlatform>JDK_1.8</netbeans.hint.jdkPlatform>
+    </properties>
+</project-shared-configuration>

dhp-workflows/dhp-usage-datasets-stats-update/pom.xml

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+
+<!--     <parent> 
+        <artifactId>dhp-workflows</artifactId >
+        <groupId>eu.dnetlib.dhp</groupId>
+        <version>1.1.7-SNAPSHOT</version>
+    </parent>
+    <groupId>eu.dnetlib</groupId> -->
+<!--     <parent>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <artifactId>dhp-workflows</artifactId>
+        <version>1.1.7-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>dhp-usage-stats-update</artifactId> -->
+    
+    <parent>
+        <artifactId>dhp-workflows</artifactId>
+        <groupId>eu.dnetlib.dhp</groupId>
+        <version>1.2.4-SNAPSHOT</version>
+        <relativePath>../</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>dhp-usage-datasets-stats-update</artifactId>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>2.1.15</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <dotGitDirectory>${project.basedir}/../.git</dotGitDirectory>
+                    <!-- more config here as you see fit -->
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.6.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins> 
+    </build>  
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <cdh.hive.version>0.13.1-cdh5.2.1</cdh.hive.version>
+        <cdh.hadoop.version>2.5.0-cdh5.2.1</cdh.hadoop.version>
+	</properties>
+    
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_2.11</artifactId>
+            <version>2.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_2.11</artifactId>
+            <version>2.4.5</version>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.json-simple</groupId>
+            <artifactId>json-simple</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+            <version>20180130</version>
+            <type>jar</type>
+        </dependency>
+		<dependency>
+			<groupId>org.apache.hive</groupId>
+			<artifactId>hive-jdbc</artifactId>
+			<version>${cdh.hive.version}</version>
+		</dependency>
+		<dependency>
+  			<groupId>org.apache.hadoop</groupId>
+  			<artifactId>hadoop-common</artifactId>
+  			<version>2.7.4</version>
+		 <type>jar</type>
+		</dependency>        
+        <dependency>
+            <groupId>eu.dnetlib.dhp</groupId>
+            <artifactId>dhp-common</artifactId>
+            <version>1.2.4-SNAPSHOT</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>com.mchange</groupId>
+            <artifactId>c3p0</artifactId>
+            <version>0.9.5.2</version>
+        </dependency>
+        <dependency>
+            <groupId>c3p0</groupId>
+            <artifactId>c3p0</artifactId>
+            <version>0.9.1.2</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.26</version>
+            <type>jar</type>
+        </dependency>
+    </dependencies>
+    <name>dhp-usage-datasets-stats-update</name>
+</project>

dhp-workflows/dhp-usage-datasets-stats-update/runworkflow.sh

@@ -0,0 +1 @@
+mvn clean package -Poozie-package,deploy,run -Dworkflow.source.dir=eu/dnetlib/dhp/oa/graph/datasetsusagestats

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ConnectDB.java

@@ -0,0 +1,148 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+
+import org.apache.log4j.Logger;
+
+/**
+ * @author D. Pierrakos
+ */
+/**
+ * @author D. Pierrakos
+ */
+import com.mchange.v2.c3p0.ComboPooledDataSource;
+
+public abstract class ConnectDB {
+
+	public static Connection DB_HIVE_CONNECTION;
+	public static Connection DB_IMPALA_CONNECTION;
+
+	private static String dbHiveUrl;
+	private static String dbImpalaUrl;
+	private static String datasetUsageStatsDBSchema;
+	private static String datasetsUsageStatsPermanentDBSchema;
+	private static String statsDBSchema;
+	private final static Logger logger = Logger.getLogger(ConnectDB.class);
+	private Statement stmt = null;
+
+	static void init() throws ClassNotFoundException {
+
+		dbHiveUrl = ExecuteWorkflow.dbHiveUrl;
+		dbImpalaUrl = ExecuteWorkflow.dbImpalaUrl;
+		datasetUsageStatsDBSchema = ExecuteWorkflow.datasetUsageStatsDBSchema;
+		datasetsUsageStatsPermanentDBSchema = ExecuteWorkflow.datasetsUsageStatsPermanentDBSchema;
+		statsDBSchema = ExecuteWorkflow.statsDBSchema;
+
+		Class.forName("org.apache.hive.jdbc.HiveDriver");
+	}
+
+	public static Connection getHiveConnection() throws SQLException {
+		if (DB_HIVE_CONNECTION != null && !DB_HIVE_CONNECTION.isClosed()) {
+			return DB_HIVE_CONNECTION;
+		} else {
+			DB_HIVE_CONNECTION = connectHive();
+
+			return DB_HIVE_CONNECTION;
+		}
+	}
+
+	public static Connection getImpalaConnection() throws SQLException {
+		if (DB_IMPALA_CONNECTION != null && !DB_IMPALA_CONNECTION.isClosed()) {
+			return DB_IMPALA_CONNECTION;
+		} else {
+			DB_IMPALA_CONNECTION = connectImpala();
+
+			return DB_IMPALA_CONNECTION;
+		}
+	}
+
+	public static String getDataSetUsageStatsDBSchema() {
+		String datePattern = "YYYYMMdd";
+		DateFormat df = new SimpleDateFormat(datePattern);
+// Get the today date using Calendar object.
+		Date today = Calendar.getInstance().getTime();
+		String todayAsString = df.format(today);
+
+		return ConnectDB.datasetUsageStatsDBSchema + "_" + todayAsString;
+	}
+
+	public static String getStatsDBSchema() {
+		return ConnectDB.statsDBSchema;
+	}
+
+	public static String getDatasetsUsagestatsPermanentDBSchema() {
+		return ConnectDB.datasetsUsageStatsPermanentDBSchema;
+	}
+
+	private static Connection connectHive() throws SQLException {
+		logger.info("trying to open Hive connection...");
+
+		ComboPooledDataSource cpds = new ComboPooledDataSource();
+		cpds.setJdbcUrl(dbHiveUrl);
+		cpds.setUser("dimitris.pierrakos");
+		cpds.setAcquireIncrement(1);
+		cpds.setMaxPoolSize(100);
+		cpds.setMinPoolSize(1);
+		cpds.setInitialPoolSize(1);
+		cpds.setMaxIdleTime(300);
+		cpds.setMaxConnectionAge(36000);
+
+		cpds.setAcquireRetryAttempts(5);
+		cpds.setAcquireRetryDelay(2000);
+		cpds.setBreakAfterAcquireFailure(false);
+
+		cpds.setCheckoutTimeout(0);
+		cpds.setPreferredTestQuery("SELECT 1");
+		cpds.setIdleConnectionTestPeriod(60);
+
+		logger.info("Opened HIVE successfully");
+
+		return cpds.getConnection();
+//		Connection connection = DriverManager.getConnection(dbHiveUrl);
+//		logger.debug("Opened Hive successfully");
+//
+//		return connection;
+
+	}
+
+	private static Connection connectImpala() throws SQLException {
+		logger.info("trying to open Impala connection...");
+		ComboPooledDataSource cpds = new ComboPooledDataSource();
+		cpds.setJdbcUrl(dbImpalaUrl);
+		cpds.setUser("dimitris.pierrakos");
+		cpds.setAcquireIncrement(1);
+		cpds.setMaxPoolSize(100);
+		cpds.setMinPoolSize(1);
+		cpds.setInitialPoolSize(1);
+		cpds.setMaxIdleTime(300);
+		cpds.setMaxConnectionAge(36000);
+
+		cpds.setAcquireRetryAttempts(5);
+		cpds.setAcquireRetryDelay(2000);
+		cpds.setBreakAfterAcquireFailure(false);
+
+		cpds.setCheckoutTimeout(0);
+		cpds.setPreferredTestQuery("SELECT 1");
+		cpds.setIdleConnectionTestPeriod(60);
+
+		logger.info("Opened Impala successfully");
+		return cpds.getConnection();
+//		Connection connection = DriverManager.getConnection(dbHiveUrl);
+//		logger.debug("Opened Impala successfully");
+//
+//		return connection;
+
+	}
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DatasetsStatsDB.java

@@ -0,0 +1,144 @@
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import java.sql.Statement;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author D. Pierrakos
+ */
+public class DatasetsStatsDB {
+
+	private String logPath;
+	private String logRepoPath;
+	private String logPortalPath;
+
+	private Statement stmt = null;
+
+	private static final Logger logger = LoggerFactory.getLogger(DatasetsStatsDB.class);
+
+	public DatasetsStatsDB(String logRepoPath, String logPortalPath) throws Exception {
+		this.logRepoPath = logRepoPath;
+		this.logPortalPath = logPortalPath;
+
+	}
+
+	public void recreateDBAndTables() throws Exception {
+		this.createDatabase();
+		this.createTables();
+	}
+
+	private void createDatabase() throws Exception {
+		try {
+			stmt = ConnectDB.getHiveConnection().createStatement();
+
+			logger.info("Dropping datasets DB: " + ConnectDB.getDataSetUsageStatsDBSchema());
+			String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema() + " CASCADE";
+			stmt.executeUpdate(dropDatabase);
+		} catch (Exception e) {
+			logger.error("Failed to drop database: " + e);
+			throw new Exception("Failed to drop database: " + e.toString(), e);
+		}
+
+		try {
+			stmt = ConnectDB.getHiveConnection().createStatement();
+
+			logger.info("Creating datacite usagestats DB: " + ConnectDB.getDataSetUsageStatsDBSchema());
+			String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema();
+			stmt.executeUpdate(createDatabase);
+
+		} catch (Exception e) {
+			logger.error("Failed to create database: " + e);
+			throw new Exception("Failed to create database: " + e.toString(), e);
+		}
+		try {
+			stmt = ConnectDB.getHiveConnection().createStatement();
+
+			logger
+				.info(
+					"Creating permanent datasets usagestats DB: " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema());
+			String createPermanentDatabase = "CREATE DATABASE IF NOT EXISTS "
+				+ ConnectDB.getDatasetsUsagestatsPermanentDBSchema();
+			stmt.executeUpdate(createPermanentDatabase);
+			logger
+				.info(
+					"Created permanent datasets usagestats DB: " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema());
+
+		} catch (Exception e) {
+			logger.error("Failed to create database: " + e);
+			throw new Exception("Failed to create database: " + e.toString(), e);
+		}
+	}
+
+	private void createTables() throws Exception {
+		try {
+			stmt = ConnectDB.getHiveConnection().createStatement();
+
+			// Create Reports table - This table should exist
+			logger.info("Creating Reports Tmp Table");
+			String sqlCreateTableDataciteReports = "CREATE TABLE IF NOT EXISTS "
+				+ ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datacitereports_tmp(reportid STRING, \n"
+				+ "	name STRING, \n"
+				+ "    source STRING,\n"
+				+ "    release STRING,\n"
+				+ "    createdby STRING,\n"
+				+ "    report_start_date STRING,\n"
+				+ "    report_end_date STRING)\n"
+				+ "    CLUSTERED BY (reportid)\n"
+				+ "	into 100 buckets stored as orc tblproperties('transactional'='true')";
+
+			stmt.executeUpdate(sqlCreateTableDataciteReports);
+			logger.info("Reports Table Created");
+
+			// Create Datasets Performance Table
+			logger.info("Creating DataSetsPerformance Tmp Table");
+			String sqlCreateTableDataSetsPerformance = "CREATE TABLE IF NOT EXISTS "
+				+ ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datasetsperformance_tmp(ds_type STRING,\n"
+				+ " ds_title STRING,\n"
+				+ " yop STRING,\n"
+				+ " dataset_type STRING, \n"
+				+ " uri STRING,\n"
+				+ " platform STRING,\n"
+				+ " publisher STRING,\n"
+				+ " publisher_id array<struct<type:STRING, value:STRING>>,\n"
+				+ " dataset_contributors array<struct<type:STRING, value:STRING>>,\n"
+				+ " period_end STRING,\n"
+				+ " period_from STRING,\n"
+				+ " access_method STRING,\n"
+				+ " metric_type STRING,\n"
+				+ " count INT,\n"
+				+ " reportid STRING)\n"
+				+ " CLUSTERED BY (ds_type)\n"
+				+ " into 100 buckets stored as orc tblproperties('transactional'='true')";
+			stmt.executeUpdate(sqlCreateTableDataSetsPerformance);
+			logger.info("DataSetsPerformance Tmp Table Created");
+
+			logger.info("Creating Datacite Reports table");
+			String createDataciteReportsTable = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datacitereports LIKE " + ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datacitereports_tmp STORED AS PARQUET";
+			stmt.executeUpdate(createDataciteReportsTable);
+			logger.info("Datacite Reports Table created");
+
+			logger.info("Creating Datasets Performance table");
+			String createDatasetPerformanceTable = "CREATE TABLE IF NOT EXISTS "
+				+ ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datasetsperformance LIKE " + ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datasetsperformance_tmp STORED AS PARQUET";
+			stmt.executeUpdate(createDatasetPerformanceTable);
+			logger.info("DatasetsPerformance Table created");
+
+			stmt.close();
+			ConnectDB.getHiveConnection().close();
+
+		} catch (Exception e) {
+			logger.error("Failed to create tables: " + e);
+			throw new Exception("Failed to create tables: " + e.toString(), e);
+		}
+	}
+
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/DownloadReportsListFromDatacite.java

@@ -0,0 +1,100 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+/**
+ * @author D.Pierrakos
+ */
+public class DownloadReportsListFromDatacite {
+
+	private String dataciteBaseURL;
+	private String dataciteReportPath;
+	private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
+
+	public DownloadReportsListFromDatacite(String dataciteBaseURL, String dataciteReportPath)
+		throws MalformedURLException, Exception {
+
+		this.dataciteBaseURL = dataciteBaseURL;
+		this.dataciteReportPath = dataciteReportPath;
+	}
+
+	public void downloadReportsList() throws ParseException {
+		StringBuilder responseStrBuilder = new StringBuilder();
+
+		Gson gson = new Gson();
+
+		try {
+			BufferedInputStream in = new BufferedInputStream(new URL(dataciteBaseURL).openStream());
+			BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
+			String inputStr;
+
+			while ((inputStr = streamReader.readLine()) != null) {
+				responseStrBuilder.append(inputStr);
+			}
+		} catch (IOException e) {
+			logger.info(e.getMessage());
+		}
+		JsonObject jsonObject = gson.fromJson(responseStrBuilder.toString(), JsonObject.class);
+		JsonArray dataArray = jsonObject.getAsJsonArray("reports");
+		ArrayList reportsList = new ArrayList();
+		for (JsonElement element : dataArray) {
+			reportsList.add(element.getAsJsonObject().get("id").getAsString());
+		}
+
+		Iterator it = reportsList.iterator();
+		while (it.hasNext()) {
+			String reportId = it.next().toString();
+			String url = dataciteBaseURL + reportId;
+
+			try {
+				BufferedInputStream in = new BufferedInputStream(new URL(url).openStream());
+				BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
+				String inputStr;
+				StringBuilder responseStrBuilder2 = new StringBuilder();
+				while ((inputStr = streamReader.readLine()) != null) {
+					responseStrBuilder2.append(inputStr);
+				}
+				FileSystem fs = FileSystem.get(new Configuration());
+				FSDataOutputStream fin = fs
+					.create(
+						new Path(dataciteReportPath + "/" + reportId + ".json"),
+						true);
+				byte[] jsonObjectRawBytes = responseStrBuilder2.toString().getBytes();
+				fin.write(jsonObjectRawBytes);
+				fin.writeChar('\n');
+
+				fin.close();
+
+				fin.close();
+			} catch (IOException e) {
+				System.out.println(e);
+			}
+		}
+	}
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ExecuteWorkflow.java

@@ -0,0 +1,71 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.BasicConfigurator;
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+
+/**
+ * @author D. Pierrakos, S. Zoupanos
+ */
+public class ExecuteWorkflow {
+
+	static String dataciteBaseURL;
+	static String dataciteReportPath;
+	static String dbHiveUrl;
+	static String dbImpalaUrl;
+	static String datasetUsageStatsDBSchema;
+	static String datasetsUsageStatsPermanentDBSchema;
+	static String statsDBSchema;
+	static boolean recreateDbAndTables;
+	static boolean datasetsEmptyDirs;
+	static boolean finalTablesVisibleToImpala;
+
+	public static void main(String args[]) throws Exception {
+
+		// Sending the logs to the console
+		BasicConfigurator.configure();
+
+		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+			IOUtils
+				.toString(
+					UsageStatsExporter.class
+						.getResourceAsStream(
+							"/eu/dnetlib/dhp/oa/graph/datasetsusagestats/export/datasets_usagestats_parameters.json")));
+		parser.parseArgument(args);
+
+		// Setting up the initial parameters
+		dataciteBaseURL = parser.get("dataciteBaseURL");
+		dataciteReportPath = parser.get("dataciteReportPath");
+		dbHiveUrl = parser.get("dbHiveUrl");
+		dbImpalaUrl = parser.get("dbImpalaUrl");
+		datasetUsageStatsDBSchema = parser.get("datasetUsageStatsDBSchema");
+		datasetsUsageStatsPermanentDBSchema = parser.get("datasetsUsageStatsPermanentDBSchema");
+		statsDBSchema = parser.get("statsDBSchema");
+
+		if (parser.get("recreateDbAndTables").toLowerCase().equals("true"))
+			recreateDbAndTables = true;
+		else
+			recreateDbAndTables = false;
+
+		if (parser.get("datasetsEmptyDirs").toLowerCase().equals("true"))
+			datasetsEmptyDirs = true;
+		else
+			datasetsEmptyDirs = false;
+
+		if (parser.get("finalTablesVisibleToImpala").toLowerCase().equals("true"))
+			finalTablesVisibleToImpala = true;
+		else
+			finalTablesVisibleToImpala = false;
+
+		UsageStatsExporter usagestatsExport = new UsageStatsExporter();
+		usagestatsExport.export();
+	}
+
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/ReadReportsListFromDatacite.java

@@ -0,0 +1,388 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * @author D.Pierrakos
+ */
+public class ReadReportsListFromDatacite {
+
+	private String dataciteReportPath;
+	private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
+
+	public ReadReportsListFromDatacite(String dataciteReportPath) throws MalformedURLException, Exception {
+
+		this.dataciteReportPath = dataciteReportPath;
+	}
+
+	public void readReports() throws Exception {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+		ArrayList<String> jsonFiles = listHdfsDir(dataciteReportPath);
+		for (String jsonFile : jsonFiles) {
+			logger.info("Reading report file " + jsonFile);
+			this.createTmpReportsTable(jsonFile);
+
+			String sqlSelectReportID = "SELECT get_json_object(json, '$.report.id')  FROM "
+				+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+			stmt.execute(sqlSelectReportID);
+			ResultSet rstmpReportID = stmt.getResultSet();
+
+			String reportID = null;
+			while (rstmpReportID.next()) {
+				reportID = rstmpReportID.getString(1);
+			}
+
+			logger.info("Checking report with id " + reportID);
+			String sqlCheckIfReportExists = "SELECT source FROM " + ConnectDB.getDataSetUsageStatsDBSchema()
+				+ ".datacitereports_tmp where reportid=?";
+			PreparedStatement stGetReportID = ConnectDB.getHiveConnection().prepareStatement(sqlCheckIfReportExists);
+			stGetReportID.setString(1, reportID);
+
+			ResultSet rsCheckIfReportExist = stGetReportID.executeQuery();
+
+			if (rsCheckIfReportExist.next()) {
+				logger.info("Report found with ID " + reportID);
+				dropTmpReportsTable();
+			} else {
+				String sqlInsertReport = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
+					+ " .datacitereports_tmp "
+					+ "SELECT\n"
+					+ "  get_json_object(json, '$.report.id') AS reportid,\n"
+					+ "  get_json_object(json, '$.report.report-header.report-name') AS name,\n"
+					+ "  get_json_object(json, '$.report.report-header.report-id') AS source,\n"
+					+ "  get_json_object(json, '$.report.report-header.release') AS release,\n"
+					+ "  get_json_object(json, '$.report.report-header.created-by\') AS createdby,\n"
+					+ "  get_json_object(json, '$.report.report-header.reporting-period.begin-date') AS fromdate,\n"
+					+ "  get_json_object(json, '$.report.report-header.reporting-period.end-date') AS todate    \n"
+					+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+				stmt.execute(sqlInsertReport);
+
+				logger.info("Report added");
+
+				logger.info("Adding datasets");
+				String sqlSelecteDatasetsArray = "SELECT get_json_object(json, '$.report.report-datasets')  FROM "
+					+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+				stmt.execute(sqlSelecteDatasetsArray);
+				ResultSet rstmpReportDatasets = stmt.getResultSet();
+
+				if (rstmpReportDatasets.next() && rstmpReportDatasets.getString(1).indexOf(',') > 0) {
+					// String[] listDatasets = rstmpReportDatasets.getString(1).split(",");
+					// String listDatasets = rstmpReportDatasets.getString(1);
+					String sqlSelectReport = "SELECT * FROM "
+						+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+					stmt.execute(sqlSelectReport);
+					ResultSet rstmpReportAll = stmt.getResultSet();
+					if (rstmpReportAll.next()) {
+						String listDatasets = rstmpReportAll.getString(1);
+						logger.info("Adding uncompressed performance for " + reportID);
+						this.readDatasetsReport(listDatasets, reportID);
+					}
+
+				}
+				logger.info("Adding gziped performance for datasets");
+				String sqlSelecteReportSubsets = "SELECT get_json_object(json, '$.report.report-subsets.gzip[0]')  FROM "
+					+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+				stmt.execute(sqlSelecteReportSubsets);
+				ResultSet rstmpReportSubsets = stmt.getResultSet();
+				if (rstmpReportSubsets.next()) {
+					String unCompressedReport = uncompressString(rstmpReportSubsets.getString(1));
+					this.readDatasetsReport(unCompressedReport, reportID);
+				}
+			}
+		}
+		this.dropTmpReportsTable();
+	}
+
+	public void readDatasetsReport(String prettyDatasetsReports, String reportId) throws Exception {
+		logger.info("Reading Datasets performance for report " + reportId);
+		logger.info("Write Performance Report To File");
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+		ObjectMapper objectMapper = new ObjectMapper();
+		JsonNode jsonNode = objectMapper.readValue(prettyDatasetsReports, JsonNode.class);
+		String datasetsReports = jsonNode.toString();
+		String report = datasetsReports
+			.replace("report-datasets", "report_datasets")
+			.replace("dataset-title", "dataset_title")
+			.replace("dataset-id", "dataset_id")
+			.replace("data-type", "data_type")
+			.replace("publisher-id", "publisher_id")
+			.replace("dataset-contributors", "dataset_contributors")
+			.replace("begin-date", "begin_date")
+			.replace("end-date", "end_date")
+			.replace("access-method", "access_method")
+			.replace("metric-type", "metric_type")
+			.replace("doi:", "");
+		FileSystem fs = FileSystem.get(new Configuration());
+		String tmpPath = dataciteReportPath + "/tmpjson";
+		FSDataOutputStream fin = fs
+			.create(new Path(dataciteReportPath + "/tmpjson/" + reportId + "_Compressed.json"), true);
+		byte[] jsonObjectRawBytes = report.getBytes();
+
+		fin.write(jsonObjectRawBytes);
+
+		fin.writeChar('\n');
+		fin.close();
+
+		logger.info("Reading Performance Report From File...");
+
+		String sqlCreateTempTableForDatasets = "CREATE TEMPORARY TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".tmpjsoncompressesed (report_datasets array<struct<dataset_id:array<struct<value:string>>,dataset_title:string, data_type:string, "
+			+ "uri:string, publisher:string, publisher_id:array<struct<type:string, value:string>>,platform:string, yop:string, "
+			+ "dataset_contributors:array<struct<type:string, value:string>>,"
+			+ "performance:array<struct<period:struct<begin_date:string,end_date:string>, "
+			+ "instance:array<struct<count:int,access_method:string,metric_type:string>>>>>>) "
+			+ "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n"
+			+ "LOCATION '" + tmpPath + "'";
+
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Adding JSON Serde jar");
+		stmt.executeUpdate("add jar /usr/share/cmf/common_jars/hive-hcatalog-core-1.1.0-cdh5.14.0.jar");
+		logger.info("Added JSON Serde jar");
+
+		logger.info("Inserting Datasets Performance");
+		stmt.execute(sqlCreateTempTableForDatasets);
+
+		String sqlInsertToDatasetsPerformance = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".datasetsperformance_tmp SELECT dataset.dataset_id[0].value ds_type, "
+			+ " dataset.dataset_title ds_title, "
+			+ " dataset.yop yop, "
+			+ " dataset.data_type dataset_type, "
+			+ " dataset.uri uri, "
+			+ " dataset.platform platform, "
+			+ " dataset.publisher publisher, "
+			+ " dataset.publisher_id publisher_id, "
+			+ " dataset.dataset_contributors dataset_contributors, "
+			+ " period.end_date period_end, "
+			+ " period.begin_date period_from, "
+			+ " performance.access_method access_method, "
+			+ " performance.metric_type metric_type, "
+			+ " performance.count count, "
+			+ "'" + reportId + "' report_id "
+			+ " FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsoncompressesed "
+			+ " LATERAL VIEW explode(report_datasets) exploded_table as dataset LATERAL VIEW explode(dataset.performance[0].instance) exploded_table2 as performance "
+			+ " LATERAL VIEW explode (array(dataset.performance[0].period)) exploded_table3 as period";
+
+		stmt.executeUpdate(sqlInsertToDatasetsPerformance);
+
+		logger.info("Datasets Performance Inserted for Report " + reportId);
+
+		stmt.execute("Drop table " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsoncompressesed");
+
+		logger.info("Datasets Report Added");
+
+	}
+
+	private ArrayList<String> listHdfsDir(String dir) throws Exception {
+
+		FileSystem hdfs = FileSystem.get(new Configuration());
+		RemoteIterator<LocatedFileStatus> Files;
+		ArrayList<String> fileNames = new ArrayList<>();
+
+		try {
+			Path exportPath = new Path(hdfs.getUri() + dir);
+			Files = hdfs.listFiles(exportPath, false);
+			while (Files.hasNext()) {
+				String fileName = Files.next().getPath().toString();
+				fileNames.add(fileName);
+			}
+
+			hdfs.close();
+		} catch (Exception e) {
+			logger.error("HDFS file path with exported data does not exist : " + new Path(hdfs.getUri() + dir));
+			throw new Exception("HDFS file path with exported data does not exist :   " + dir, e);
+		}
+
+		return fileNames;
+	}
+
+	private String readHDFSFile(String filename) throws Exception {
+		String result;
+		try {
+
+			FileSystem fs = FileSystem.get(new Configuration());
+			// log.info("reading file : " + filename);
+
+			BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(filename))));
+
+			StringBuilder sb = new StringBuilder();
+			String line = br.readLine();
+
+			while (line != null) {
+				sb.append(line);
+				// sb.append(line);
+				line = br.readLine();
+			}
+			// uncompressedReport = sb.toString().replace("][{\"idSite\"", ",{\"idSite\"");
+			result = sb.toString().trim();
+			// fs.close();
+		} catch (Exception e) {
+			throw new Exception(e);
+		}
+
+		return result;
+	}
+
+	public static String uncompressString(String zippedBase64Str)
+		throws IOException {
+		String uncompressedReport = null;
+
+		byte[] bytes = Base64.getDecoder().decode(zippedBase64Str);
+		GZIPInputStream zi = null;
+		try {
+			zi = new GZIPInputStream(new ByteArrayInputStream(bytes));
+			uncompressedReport = IOUtils.toString(zi);
+		} finally {
+			IOUtils.closeQuietly(zi);
+		}
+		logger.info("Report Succesfully Uncompressed...");
+		return uncompressedReport;
+	}
+
+	private void createTmpReportsTable(String jsonFile) throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		dropTmpReportsTable();
+		String createTmpTable = "CREATE TEMPORARY TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".tmpjsonToTable (json STRING)";
+		stmt.executeUpdate(createTmpTable);
+		logger.info("Temporary Table for Json Report Created");
+
+		String insertJsonReport = "LOAD DATA INPATH '" + jsonFile + "' INTO TABLE "
+			+ ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+		stmt.execute(insertJsonReport);
+		logger.info("JSON Report File inserted to tmpjsonToTable Table");
+	}
+
+	private void dropTmpReportsTable() throws SQLException {
+		logger.info("Dropping tmpjson Table");
+		String dropTmpTable = "DROP TABLE IF EXISTS " + ConnectDB.getDataSetUsageStatsDBSchema() + ".tmpjsonToTable";
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		stmt.executeUpdate(dropTmpTable);
+		logger.info("Dropped Table for Json Report Table");
+
+	}
+
+	public void createUsageStatisticsTable() throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+
+		logger.info("Updating Datacite Reports table");
+		String createDataciteReportsTable = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".datacitereports "
+			+ "SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports_tmp";
+		stmt.executeUpdate(createDataciteReportsTable);
+		logger.info("Datacite Reports Table updated");
+
+		logger.info("Updating Datasets Performance table");
+		String createDatasetPerformanceTable = "INSERT INTO " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".datasetsperformance "
+			+ "SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance_tmp";
+		stmt.executeUpdate(createDatasetPerformanceTable);
+		logger.info("DatasetsPerformance Table updated");
+
+		logger.info("Creating Downloads Stats table");
+		String createDownloadsTable = "CREATE TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".datacite_downloads STORED AS PARQUET as "
+			+ "SELECT 'Datacite' source, d.id repository_id, od.id result_id, regexp_replace(substring(string(period_end),0,7),'-','/') date, count, '0' openaire "
+			+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance "
+			+ "JOIN " + ConnectDB.getStatsDBSchema() + ".datasource d on name=platform "
+			+ "JOIN " + ConnectDB.getStatsDBSchema() + ".result_oids od on string(ds_type)=od.oid "
+			+ "where metric_type='total-dataset-requests' ";
+		stmt.executeUpdate(createDownloadsTable);
+		logger.info("Downloads Stats table created");
+
+		logger.info("Creating Views Stats table");
+		String createViewsTable = "CREATE TABLE " + ConnectDB.getDataSetUsageStatsDBSchema()
+			+ ".datacite_views STORED AS PARQUET as "
+			+ "SELECT 'Datacite' source, d.id repository_id, od.id result_id, regexp_replace(substring(string(period_end),0,7),'-','/') date, count, '0' openaire "
+			+ "FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance "
+			+ "JOIN " + ConnectDB.getStatsDBSchema() + ".datasource d on name=platform "
+			+ "JOIN " + ConnectDB.getStatsDBSchema() + ".result_oids od on string(ds_type)=od.oid "
+			+ "where metric_type='total-dataset-investigations' ";
+		stmt.executeUpdate(createViewsTable);
+		logger.info("Views Stats table created");
+
+		logger.info("Building Permanent Datasets Usage Stats DB");
+
+		logger.info("Dropping view datacitereports on permanent datacite usagestats DB");
+		String sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped view datacitereports on permanent datacite usagestats DB");
+
+		logger.info("Create view datacitereports on permanent datacite usagestats DB");
+		sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports"
+			+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports";
+		stmt.executeUpdate(sql);
+		logger.info("Created view datacitereports on permanent datasets usagestats DB");
+
+		logger.info("Dropping view datasetsperformance on permanent datacite usagestats DB");
+		sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped view datasetsperformance on permanent datacite usagestats DB");
+
+		logger.info("Create view datasetsperformance on permanent datacite usagestats DB");
+		sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance"
+			+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance";
+		stmt.executeUpdate(sql);
+		logger.info("Created view datasetsperformance on permanent datasets usagestats DB");
+
+		logger.info("Dropping view datacite_views on permanent datacite usagestats DB");
+		sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped view datacite_views on permanent datacite usagestats DB");
+
+		logger.info("Create view datacite_views on permanent datacite usagestats DB");
+		sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views"
+			+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Created view datacite_views on permanent datasets usagestats DB");
+
+		logger.info("Dropping view datacite_downloads on permanent datacite usagestats DB");
+		sql = "DROP VIEW IF EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped view datacite_downloads on permanent datacite usagestats DB");
+
+		logger.info("Create view datacite_downloads on permanent datacite usagestats DB");
+		sql = "CREATE VIEW IF NOT EXISTS " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads"
+			+ " AS SELECT * FROM " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Created view datacite_downloads on permanent datasets usagestats DB");
+
+		stmt.close();
+		ConnectDB.getHiveConnection().close();
+		logger.info("Completed Building Permanent Datasets Usage Stats DB");
+	}
+
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/java/eu/dnetlib/oa/graph/datasetsusagestats/export/UsageStatsExporter.java

@@ -0,0 +1,117 @@
+
+package eu.dnetlib.oa.graph.datasetsusagestats.export;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Main class for downloading and processing Usage statistics
+ *
+ * @author D. Pierrakos, S. Zoupanos
+ */
+public class UsageStatsExporter {
+
+	private Statement stmt = null;
+
+	public UsageStatsExporter() {
+
+	}
+
+	private static final Logger logger = LoggerFactory.getLogger(UsageStatsExporter.class);
+
+	private void reCreateLogDirs() throws IllegalArgumentException, IOException {
+		FileSystem dfs = FileSystem.get(new Configuration());
+
+		logger.info("Deleting Log directory: " + ExecuteWorkflow.dataciteReportPath);
+		dfs.delete(new Path(ExecuteWorkflow.dataciteReportPath), true);
+
+		logger.info("Creating Log directory: " + ExecuteWorkflow.dataciteReportPath);
+		dfs.mkdirs(new Path(ExecuteWorkflow.dataciteReportPath));
+
+		logger.info("Creating tmp directory: " + ExecuteWorkflow.dataciteReportPath + " " + "/tmpjson/");
+		dfs.mkdirs(new Path(ExecuteWorkflow.dataciteReportPath + "/tmpjson/"));
+
+	}
+
+	public void export() throws Exception {
+
+		logger.info("Initialising DB properties");
+		ConnectDB.init();
+		ConnectDB.getHiveConnection();
+
+		if (ExecuteWorkflow.recreateDbAndTables) {
+			DatasetsStatsDB datasetsDB = new DatasetsStatsDB("", "");
+			datasetsDB.recreateDBAndTables();
+		}
+		logger.info("Initializing the download logs module");
+		DownloadReportsListFromDatacite downloadReportsListFromDatacite = new DownloadReportsListFromDatacite(
+			ExecuteWorkflow.dataciteBaseURL,
+			ExecuteWorkflow.dataciteReportPath);
+
+		if (ExecuteWorkflow.datasetsEmptyDirs) {
+			logger.info("Downloading Reports List From Datacite");
+			this.reCreateLogDirs();
+			downloadReportsListFromDatacite.downloadReportsList();
+			logger.info("Reports List has been downloaded");
+		}
+
+		ReadReportsListFromDatacite readReportsListFromDatacite = new ReadReportsListFromDatacite(
+			ExecuteWorkflow.dataciteReportPath);
+		logger.info("Store Reports To DB");
+		readReportsListFromDatacite.readReports();
+		logger.info("Reports Stored To DB");
+		readReportsListFromDatacite.createUsageStatisticsTable();
+
+		// Make the tables available to Impala
+		if (ExecuteWorkflow.finalTablesVisibleToImpala) {
+			logger.info("Making tables visible to Impala");
+			invalidateMetadata();
+		}
+
+		logger.info("End");
+	}
+
+	private void invalidateMetadata() throws SQLException {
+		Statement stmt = null;
+
+		stmt = ConnectDB.getImpalaConnection().createStatement();
+
+		String sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datacitereports";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDataSetUsageStatsDBSchema() + ".datasetsperformance";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datacitereports";
+		stmt.executeUpdate(sql);
+
+		sql = "INVALIDATE METADATA " + ConnectDB.getDatasetsUsagestatsPermanentDBSchema() + ".datasetsperformance";
+		stmt.executeUpdate(sql);
+
+		stmt.close();
+		try {
+			ConnectDB.getHiveConnection().close();
+		} catch (Exception e) {
+			logger.info("Message at the end :" + e.getMessage());
+		}
+	}
+}

dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/export/datasets_usagestats_parameters.json

@@ -0,0 +1,62 @@
+[
+    {
+        "paramName": "dbu",
+        "paramLongName": "dataciteBaseURL",
+        "paramDescription": "URL of Datacite Reports Endpoint",
+        "paramRequired": true
+    },
+    {
+        "paramName": "drp",
+        "paramLongName": "dataciteReportPath",
+        "paramDescription": "Path for Datacite Reports",
+        "paramRequired": true
+    },
+    {
+        "paramName": "dbhu",
+        "paramLongName": "dbHiveUrl",
+        "paramDescription": "activate tranform-only mode. Only apply transformation step",
+        "paramRequired": true
+    },
+    {
+        "paramName": "dbiu",
+        "paramLongName": "dbImpalaUrl",
+        "paramDescription": "activate tranform-only mode. Only apply transformation step",
+        "paramRequired": true
+    },
+    {
+        "paramName": "dusdbs",
+        "paramLongName": "datasetUsageStatsDBSchema",
+        "paramDescription": "activate tranform-only mode. Only apply transformation step",
+        "paramRequired": true
+    },
+    {
+        "paramName": "uspdbs",
+        "paramLongName": "datasetsUsageStatsPermanentDBSchema",
+        "paramDescription": "activate tranform-only mode. Only apply transformation step",
+        "paramRequired": true
+    }, 
+    {
+        "paramName": "sdbs",
+        "paramLongName": "statsDBSchema",
+        "paramDescription": "activate tranform-only mode. Only apply transformation step",
+        "paramRequired": true
+    },
+    {
+        "paramName": "rdbt",
+        "paramLongName": "recreateDbAndTables",
+        "paramDescription": "Re-create database and initial tables?",
+        "paramRequired": true
+    },
+    {
+        "paramName": "pwed",
+        "paramLongName": "datasetsEmptyDirs",
+        "paramDescription": "Empty piwik directories?",
+        "paramRequired": true
+    },
+    {
+        "paramName": "ftvi",
+        "paramLongName": "finalTablesVisibleToImpala",
+        "paramDescription": "Make the dataset_usage_stats, visible to Impala",
+        "paramRequired": true
+    }
+]

dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/config-default.xml

@@ -0,0 +1,38 @@
+<configuration>
+    <property>
+        <name>jobTracker</name>
+        <value>${jobTracker}</value>
+    </property>
+    <property>
+        <name>nameNode</name>
+        <value>${nameNode}</value>
+    </property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>oozie.action.sharelib.for.spark</name>
+        <value>spark2</value>
+    </property>
+    <property>
+        <name>hiveMetastoreUris</name>
+        <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
+    </property>
+    <property>
+        <name>hiveJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1</value>
+    </property>
+    <property>
+        <name>impalaJdbcUrl</name>
+        <value>jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl;</value>
+    </property>
+	<property>
+		<name>oozie.wf.workflow.notification.url</name>
+		<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
+	</property>
+    <property>
+        <name>oozie.use.system.libpath</name>
+        <value>true</value>
+    </property>
+</configuration>

dhp-workflows/dhp-usage-datasets-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/datasetsusagestats/oozie_app/workflow.xml

@@ -0,0 +1,72 @@
+<workflow-app name="Datacite Datasets Usage Stats" xmlns="uri:oozie:workflow:0.5">
+    <parameters>
+        <property>
+            <name>hiveMetastoreUris</name>
+            <description>Hive server metastore URIs</description>
+        </property>
+        <property>
+            <name>hiveJdbcUrl</name>
+            <description>Hive server jdbc url</description>
+        </property>
+        <property>
+            <name>impalaJdbcUrl</name>
+            <description>Impala server jdbc url</description>
+        </property>
+    </parameters>
+
+    <global>
+        <job-tracker>${jobTracker}</job-tracker>
+        <name-node>${nameNode}</name-node>
+        <configuration>
+            <property>
+                <name>hive.metastore.uris</name>
+                <value>${hiveMetastoreUris}</value>
+            </property>
+            <property>
+                <name>mapreduce.job.queuename</name>
+                <value>${queueName}</value>
+            </property>
+            <property>
+                <name>oozie.launcher.mapred.job.queue.name</name>
+                <value>${oozieLauncherQueueName}</value>
+            </property>
+        </configuration>
+    </global>
+
+    <start to="Step1"/>
+
+    <kill name="Kill">
+        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+
+    <action name='Step1'>
+        <java>
+            <main-class>eu.dnetlib.oa.graph.datasetsusagestats.export.ExecuteWorkflow</main-class>
+            <arg>--dataciteBaseURL</arg>
+            <arg>${dataciteBaseURL}</arg>
+            <arg>--dataciteReportPath</arg>
+            <arg>${dataciteReportPath}</arg>
+            <arg>--dbHiveUrl</arg>
+            <arg>${hiveJdbcUrl}</arg>
+            <arg>--dbImpalaUrl</arg>
+            <arg>${impalaJdbcUrl}</arg>
+            <arg>--datasetUsageStatsDBSchema</arg>
+            <arg>${datasetUsageStatsDBSchema}</arg>
+            <arg>--datasetsUsageStatsPermanentDBSchema</arg>
+            <arg>${datasetsUsageStatsPermanentDBSchema}</arg>           
+            <arg>--statsDBSchema</arg>
+            <arg>${statsDBSchema}</arg>
+            <arg>--recreateDbAndTables</arg>
+            <arg>${recreateDbAndTables}</arg>
+            <arg>--datasetsEmptyDirs</arg>
+            <arg>${datasetsEmptyDirs}</arg>
+            <arg>--finalTablesVisibleToImpala</arg>
+            <arg>${finalTablesVisibleToImpala}</arg>
+            <capture-output/>
+        </java>
+        <ok to="End" />
+        <error to="Kill" />
+    </action>
+    
+    <end name="End"/>
+</workflow-app>

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/ExecuteWorkflow.java

@@ -65,6 +65,8 @@ public class ExecuteWorkflow {
 
 	static int numberOfDownloadThreads;
 
+	static int b2SSHAREID;
+
 	public static void main(String args[]) throws Exception {
 
 		// Sending the logs to the console
@@ -196,6 +198,8 @@ public class ExecuteWorkflow {
 
 		numberOfDownloadThreads = Integer.parseInt(parser.get("numberOfDownloadThreads"));
 
+		b2SSHAREID = Integer.parseInt(parser.get("b2shareID"));
+
 		UsageStatsExporter usagestatsExport = new UsageStatsExporter();
 		usagestatsExport.export();
 		// usagestatsExport.createdDBWithTablesOnly();

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs.java

@@ -191,7 +191,7 @@ public class PiwikDownloadLogs {
 		ResultSet rs = statement
 			.executeQuery(
 				"SELECT distinct piwik_id from " + ConnectDB.getStatsDBSchema()
-					+ ".datasource where piwik_id is not null and piwik_id <> 0 order by piwik_id");
+					+ ".datasource where piwik_id is not null and piwik_id <> 0 and piwik_id <> 196 order by piwik_id");
 
 		// Getting all the piwikids in a list for logging reasons & limitting the list
 		// to the max number of piwikids

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikDownloadLogs_B2SHARE.java

@@ -0,0 +1,204 @@
+
+package eu.dnetlib.oa.graph.usagerawdata.export;
+
+import java.io.*;
+import java.net.URL;
+import java.net.URLConnection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author D. Pierrakos
+ */
+public class PiwikDownloadLogs_B2SHARE {
+
+	private final String piwikUrl;
+	private Date startDate;
+	private final String tokenAuth;
+
+	/*
+	 * The Piwik's API method
+	 */
+	private final String APImethod = "?module=API&method=Live.getLastVisitsDetails";
+	private final String format = "&format=json";
+
+	private static final Logger logger = LoggerFactory.getLogger(PiwikDownloadLogs_B2SHARE.class);
+
+	public PiwikDownloadLogs_B2SHARE(String piwikUrl, String tokenAuth) {
+		this.piwikUrl = piwikUrl;
+		this.tokenAuth = tokenAuth;
+
+	}
+
+	private String getPiwikLogUrl() {
+		return "https://" + piwikUrl + "/";
+	}
+
+	private String getJson(String url) throws Exception {
+		try {
+			logger.debug("Connecting to download the JSON: " + url);
+			URL website = new URL(url);
+			URLConnection connection = website.openConnection();
+
+			StringBuilder response;
+			try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+				response = new StringBuilder();
+				String inputLine;
+				while ((inputLine = in.readLine()) != null) {
+					response.append(inputLine);
+				}
+			}
+			return response.toString();
+		} catch (Exception e) {
+			logger.error("Failed to get URL: " + url + " Exception: " + e);
+			throw new Exception("Failed to get URL: " + url + " Exception: " + e.toString(), e);
+		}
+	}
+
+	public void GetOpenAIREB2SHARELogs(String repoLogsPath) throws Exception {
+
+		Statement statement = ConnectDB.getHiveConnection().createStatement();
+		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+
+		List<Integer> piwikIdToVisit = new ArrayList<Integer>();
+		piwikIdToVisit.add(ExecuteWorkflow.b2SSHAREID);
+		logger.info("B2SHARE piwikId for download: " + piwikIdToVisit);
+
+		if (ExecuteWorkflow.numberOfPiwikIdsToDownload > 0
+			&& ExecuteWorkflow.numberOfPiwikIdsToDownload <= piwikIdToVisit.size()) {
+			logger.info("Trimming piwikIds list to the size of: " + ExecuteWorkflow.numberOfPiwikIdsToDownload);
+			piwikIdToVisit = piwikIdToVisit.subList(0, ExecuteWorkflow.numberOfPiwikIdsToDownload);
+		}
+
+		logger.info("Downloading for the followins piwikIds: " + piwikIdToVisit);
+
+		// ExecutorService executor = Executors.newFixedThreadPool(ExecuteWorkflow.numberOfDownloadThreads);
+		for (int siteId : piwikIdToVisit) {
+			// Setting the starting period
+			Calendar start = (Calendar) ExecuteWorkflow.startingLogPeriod.clone();
+			logger.info("Starting period for log download: " + sdf.format(start.getTime()));
+
+			// Setting the ending period (last day of the month)
+			// Calendar end = (Calendar) ExecuteWorkflow.endingLogPeriod.clone();
+			Calendar end = Calendar.getInstance();
+			end.add(Calendar.DAY_OF_MONTH, -1);
+			// end.add(Calendar.MONTH, +1);
+//			end.add(Calendar.DAY_OF_MONTH, -1);
+			logger.info("Ending period for log download: " + sdf.format(end.getTime()));
+
+			logger.info("Now working on piwikId: " + siteId);
+
+			PreparedStatement st = ConnectDB.DB_HIVE_CONNECTION
+				.prepareStatement(
+					"SELECT max(timestamp) FROM " + ConnectDB.getUsageStatsDBSchema()
+						+ ".piwiklog WHERE source=?");
+			st.setInt(1, siteId);
+			Date dateMax = null;
+			ResultSet rs_date = st.executeQuery();
+			while (rs_date.next()) {
+				logger.info("Found max date: " + rs_date.getString(1) + " for repository " + siteId);
+
+				if (rs_date.getString(1) != null && !rs_date.getString(1).equals("null")
+					&& !rs_date.getString(1).equals("")) {
+					start.setTime(sdf.parse(rs_date.getString(1)));
+					dateMax = sdf.parse(rs_date.getString(1));
+				}
+			}
+			rs_date.close();
+
+			for (Calendar currDay = (Calendar) start.clone(); currDay.before(end); currDay.add(Calendar.DATE, 1)) {
+				// logger.info("Date used " + currDay.toString());
+				// Runnable worker = new WorkerThread(currDay, siteId, repoLogsPath, portalLogPath, portalMatomoID);
+				// executor.execute(worker);// calling execute method of ExecutorService
+				logger.info("Date used " + currDay.getTime().toString());
+
+				if (dateMax != null && currDay.getTime().compareTo(dateMax) <= 0) {
+					logger.info("Date found in logs " + dateMax + " and not downloanding Matomo logs for " + siteId);
+				} else {
+					GetOpenAIRELogsB2SHAREForDate(currDay, siteId, repoLogsPath);
+				}
+
+			}
+		}
+		// executor.shutdown();
+		// while (!executor.isTerminated()) {
+		// }
+		// System.out.println("Finished all threads");
+	}
+
+	public void GetOpenAIRELogsB2SHAREForDate(Calendar currDay, int siteId, String repoLogsPath) throws Exception {
+		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+
+		Date date = currDay.getTime();
+		logger.info("Downloading logs for repoid " + siteId + " and for " + sdf.format(date));
+
+		String period = "&period=day&date=" + sdf.format(date);
+		String outFolder = repoLogsPath;
+
+		String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + siteId + period + format
+			+ "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;
+		String content = "";
+
+		int i = 0;
+
+		JSONParser parser = new JSONParser();
+		StringBuffer totalContent = new StringBuffer();
+		FileSystem fs = FileSystem.get(new Configuration());
+
+		do {
+			int writtenBytes = 0;
+			String apiUrl = baseApiUrl;
+
+			if (i > 0) {
+				apiUrl += "&filter_offset=" + (i * 1000);
+			}
+
+			content = getJson(apiUrl);
+			if (content.length() == 0 || content.equals("[]")) {
+				break;
+			}
+
+			FSDataOutputStream fin = fs
+				.create(
+					new Path(outFolder + "/" + siteId + "_Piwiklog" + sdf.format((date)) + "_offset_" + i
+						+ ".json"),
+					true);
+			JSONArray jsonArray = (JSONArray) parser.parse(content);
+			for (Object aJsonArray : jsonArray) {
+				JSONObject jsonObjectRaw = (JSONObject) aJsonArray;
+				byte[] jsonObjectRawBytes = jsonObjectRaw.toJSONString().getBytes();
+				fin.write(jsonObjectRawBytes);
+				fin.writeChar('\n');
+
+				writtenBytes += jsonObjectRawBytes.length + 1;
+			}
+
+			fin.close();
+			System.out
+				.println(
+					Thread.currentThread().getName() + " (Finished writing) Wrote " + writtenBytes
+						+ " bytes. Filename: " + siteId + "_Piwiklog" + sdf.format((date)) + "_offset_" + i
+						+ ".json");
+
+			i++;
+		} while (true);
+
+		fs.close();
+	}
+}

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB.java

@@ -179,6 +179,10 @@ public class PiwikStatsDB {
 			createPedocsOldUsageData();
 			logger.info("Pedocs Tables Created");
 
+			logger.info("Create Datacite Tables");
+			createDatasetsUsageData();
+			logger.info("Datacite Tables Created");
+
 		} catch (Exception e) {
 			logger.error("Failed to process logs: " + e);
 			throw new Exception("Failed to process logs: " + e.toString(), e);
@@ -281,6 +285,7 @@ public class PiwikStatsDB {
 
 		// clean view double clicks
 		logger.info("Cleaning action double clicks");
+		ConnectDB.getHiveConnection().setAutoCommit(false);
 		sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp "
 			+ "WHERE EXISTS (\n"
 			+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
@@ -750,6 +755,16 @@ public class PiwikStatsDB {
 		stmt.executeUpdate(sql);
 		logger.info("Dropped sarc_sushilogtmp_json_non_array");
 
+		logger.info("Dropping piwiklogb2sharetmp");
+		sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped piwiklogb2sharetmp");
+
+		logger.info("Dropping piwiklog_b2share_tmp_json");
+		sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog_b2share_tmp_json";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped piwiklog_b2share_tmp_json");
+
 		stmt.close();
 		ConnectDB.getHiveConnection().close();
 
@@ -832,4 +847,32 @@ public class PiwikStatsDB {
 		logger.info("PeDocs Old Downloads Table created");
 
 	}
+
+	public void createDatasetsUsageData() throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Dropping datacite_views");
+		String sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped datacite_views");
+
+		logger.info("Dropping datacite_downloads");
+		sql = "DROP TABLE " + ConnectDB.getUsageStatsDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Dropped datacite_downloads");
+
+		logger.info("Creating Datasets Views Table");
+		sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".datacite_views as select * from openaire_prod_datacite_usage_stats.datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Datasets Views Table created");
+
+		logger.info("Creating Datasets Downloads Table");
+		sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".datacite_downloads as select * from openaire_prod_datacite_usage_stats.datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Datasets Downloads Table created");
+
+	}
 }

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/PiwikStatsDB_B2SHARE.java

@@ -0,0 +1,304 @@
+
+package eu.dnetlib.oa.graph.usagerawdata.export;
+
+import java.io.*;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author D. Pierrakos, S. Zoupanos
+ */
+public class PiwikStatsDB_B2SHARE {
+
+	private String logPath;
+	private String logRepoPath;
+	private String logPortalPath;
+
+	private Statement stmt = null;
+
+	private static final Logger logger = LoggerFactory.getLogger(PiwikStatsDB_B2SHARE.class);
+
+	private String CounterRobotsURL;
+	private ArrayList robotsList;
+
+	public PiwikStatsDB_B2SHARE(String logRepoPath, String logPortalPath) throws Exception {
+		this.logRepoPath = logRepoPath;
+		this.logPortalPath = logPortalPath;
+
+	}
+
+	public ArrayList getRobotsList() {
+		return robotsList;
+	}
+
+	public void setRobotsList(ArrayList robotsList) {
+		this.robotsList = robotsList;
+	}
+
+	public String getCounterRobotsURL() {
+		return CounterRobotsURL;
+	}
+
+	public void setCounterRobotsURL(String CounterRobotsURL) {
+		this.CounterRobotsURL = CounterRobotsURL;
+	}
+
+	public void processB2SHARELogs() throws Exception {
+		try {
+
+			logger.info("Processing B2SHARE logs");
+			processLog();
+			logger.info("B2SHARE logs process done");
+
+			logger.info("Removing double clicks from B2SHARE logs");
+			removeDoubleClicks();
+			logger.info("Removing double clicks from B2SHARE logs done");
+
+			logger.info("Updating Production Tables");
+			updateProdTables();
+			logger.info("Updated Production Tables");
+
+		} catch (Exception e) {
+			logger.error("Failed to process logs: " + e);
+			throw new Exception("Failed to process logs: " + e.toString(), e);
+		}
+	}
+
+	public void processLog() throws Exception {
+
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Adding JSON Serde jar");
+		stmt.executeUpdate("add jar /usr/share/cmf/common_jars/hive-hcatalog-core-1.1.0-cdh5.14.0.jar");
+		logger.info("Added JSON Serde jar");
+
+		logger.info("Dropping piwiklog_b2share_tmp_json table");
+		String drop_piwiklogtmp_json = "DROP TABLE IF EXISTS "
+			+ ConnectDB.getUsageStatsDBSchema()
+			+ ".piwiklog_b2share_tmp_json";
+		stmt.executeUpdate(drop_piwiklogtmp_json);
+		logger.info("Dropped piwiklog_b2share_tmp_json table");
+
+		logger.info("Creating piwiklog_b2share_tmp_json");
+		String create_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS "
+			+ ConnectDB.getUsageStatsDBSchema()
+			+ ".piwiklog_b2share_tmp_json(\n"
+			+ "	`idSite` STRING,\n"
+			+ "	`idVisit` STRING,\n"
+			+ "	`country` STRING,\n"
+			+ "	`referrerName` STRING,\n"
+			+ "	`browser` STRING,\n"
+			+ "	`actionDetails` ARRAY<\n"
+			+ "						struct<\n"
+			+ "							type: STRING,\n"
+			+ "							url: STRING,\n"
+			+ "							eventAction: STRING,\n"
+			+ "							eventName: STRING,\n"
+			+ "							timestamp: String\n"
+			+ "							>\n"
+			+ "						>\n"
+			+ ")\n"
+			+ "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n"
+			+ "LOCATION '" + ExecuteWorkflow.repoLogPath + "'\n"
+			+ "TBLPROPERTIES (\"transactional\"=\"false\")";
+		stmt.executeUpdate(create_piwiklogtmp_json);
+		logger.info("Created piwiklog_b2share_tmp_json");
+
+		logger.info("Dropping piwiklogtmp table");
+		String drop_piwiklogtmp = "DROP TABLE IF EXISTS "
+			+ ConnectDB.getUsageStatsDBSchema()
+			+ ".piwiklogtmp";
+		stmt.executeUpdate(drop_piwiklogtmp);
+		logger.info("Dropped piwiklogtmp");
+
+		logger.info("Creating piwiklogb2sharetmp");
+		String create_piwiklogtmp = "CREATE TABLE "
+			+ ConnectDB.getUsageStatsDBSchema()
+			+ ".piwiklogb2sharetmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, "
+			+ "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING)  "
+			+ "clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true')";
+		stmt.executeUpdate(create_piwiklogtmp);
+		logger.info("Created piwiklogb2sharetmp");
+
+		logger.info("Inserting into piwiklogb2sharetmp");
+		String insert_piwiklogtmp = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
+			+ "SELECT DISTINCT cast(idSite as BIGINT) as source, idVisit  as id_Visit, country, "
+			+ "actiondetail.eventAction as action, actiondetail.url as url, "
+			+ "actiondetail.eventName as entity_id, "
+			+ "'repItem' as source_item_type, from_unixtime(cast(actiondetail.timestamp as BIGINT)) as timestamp, "
+			+ "referrerName as referrer_name, browser as agent\n"
+			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog_b2share_tmp_json\n"
+			+ "LATERAL VIEW explode(actiondetails) actiondetailsTable AS actiondetail";
+		stmt.executeUpdate(insert_piwiklogtmp);
+		logger.info("Inserted into piwiklogb2sharetmp");
+
+		stmt.close();
+	}
+
+	public void removeDoubleClicks() throws Exception {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Cleaning download double clicks");
+		// clean download double clicks
+		String sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
+			+ "WHERE EXISTS (\n"
+			+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
+			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p1, "
+			+ ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p2\n"
+			+ "WHERE p1.source=p2.source AND p1.id_visit=p2.id_visit AND p1.entity_id=p2.entity_id \n"
+			+ "AND p1.action=p2.action AND p1.action='download' AND p1.timestamp!=p2.timestamp \n"
+			+ "AND p1.timestamp<p2.timestamp AND ((unix_timestamp(p2.timestamp)-unix_timestamp(p1.timestamp))/60)<30 \n"
+			+ "AND piwiklogb2sharetmp.source=p1.source AND piwiklogb2sharetmp.id_visit=p1.id_visit \n"
+			+ "AND piwiklogb2sharetmp.action=p1.action AND piwiklogb2sharetmp.entity_id=p1.entity_id AND piwiklogb2sharetmp.timestamp=p1.timestamp)";
+		stmt.executeUpdate(sql);
+		logger.info("Cleaned download double clicks");
+
+		// clean view double clicks
+		logger.info("Cleaning action double clicks");
+		sql = "DELETE from " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
+			+ "WHERE EXISTS (\n"
+			+ "SELECT DISTINCT p1.source, p1.id_visit, p1.action, p1.entity_id, p1.timestamp \n"
+			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p1, "
+			+ ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp p2\n"
+			+ "WHERE p1.source=p2.source AND p1.id_visit=p2.id_visit AND p1.entity_id=p2.entity_id \n"
+			+ "AND p1.action=p2.action AND p1.action='action' AND p1.timestamp!=p2.timestamp \n"
+			+ "AND p1.timestamp<p2.timestamp AND (unix_timestamp(p2.timestamp)-unix_timestamp(p1.timestamp))<10 \n"
+			+ "AND piwiklogb2sharetmp.source=p1.source AND piwiklogb2sharetmp.id_visit=p1.id_visit \n"
+			+ "AND piwiklogb2sharetmp.action=p1.action AND piwiklogb2sharetmp.entity_id=p1.entity_id AND piwiklogb2sharetmp.timestamp=p1.timestamp)";
+		stmt.executeUpdate(sql);
+		logger.info("Cleaned action double clicks");
+		stmt.close();
+	}
+
+	private void updateProdTables() throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Updating recordview to action piwiklog");
+		String sqlUpdateAction = "UPDATE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
+			+ "set action='action' where action='recordview'";
+		stmt.executeUpdate(sqlUpdateAction);
+
+		logger.info("Updating fileDownload to download piwiklog");
+		String sqlUpdateDownload = "UPDATE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp "
+			+ "set action='download' where action='filedownload'";
+		stmt.executeUpdate(sqlUpdateDownload);
+
+		logger.info("Inserting B2SHARE data to piwiklog");
+		String sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog "
+			+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogb2sharetmp";
+		stmt.executeUpdate(sql);
+
+		stmt.close();
+		ConnectDB.getHiveConnection().close();
+
+	}
+
+	private ArrayList<String> listHdfsDir(String dir) throws Exception {
+
+		FileSystem hdfs = FileSystem.get(new Configuration());
+		RemoteIterator<LocatedFileStatus> Files;
+		ArrayList<String> fileNames = new ArrayList<>();
+
+		try {
+			Path exportPath = new Path(hdfs.getUri() + dir);
+			Files = hdfs.listFiles(exportPath, false);
+			while (Files.hasNext()) {
+				String fileName = Files.next().getPath().toString();
+				fileNames.add(fileName);
+			}
+
+			hdfs.close();
+		} catch (Exception e) {
+			logger.error("HDFS file path with exported data does not exist : " + new Path(hdfs.getUri() + logPath));
+			throw new Exception("HDFS file path with exported data does not exist :   " + logPath, e);
+		}
+
+		return fileNames;
+	}
+
+	private String readHDFSFile(String filename) throws Exception {
+		String result;
+		try {
+
+			FileSystem fs = FileSystem.get(new Configuration());
+			// log.info("reading file : " + filename);
+
+			BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(filename))));
+
+			StringBuilder sb = new StringBuilder();
+			String line = br.readLine();
+
+			while (line != null) {
+				if (!line.equals("[]")) {
+					sb.append(line);
+				}
+				// sb.append(line);
+				line = br.readLine();
+			}
+			result = sb.toString().replace("][{\"idSite\"", ",{\"idSite\"");
+			if (result.equals("")) {
+				result = "[]";
+			}
+
+			// fs.close();
+		} catch (Exception e) {
+			logger.error(e.getMessage());
+			throw new Exception(e);
+		}
+
+		return result;
+	}
+
+	private Connection getConnection() throws SQLException {
+		return ConnectDB.getHiveConnection();
+	}
+
+	public void createPedocsOldUsageData() throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Creating PeDocs Old Views Table");
+		String sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".pedocsoldviews as select * from default.pedocsviews";
+		stmt.executeUpdate(sql);
+		logger.info("PeDocs Old Views Table created");
+
+		logger.info("Creating PeDocs Old Downloads Table");
+		sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".pedocsolddownloads as select * from default.pedocsdownloads";
+		stmt.executeUpdate(sql);
+		logger.info("PeDocs Old Downloads Table created");
+
+	}
+
+	public void createDatasetsUsageData() throws SQLException {
+		Statement stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		logger.info("Creating Datasets Views Table");
+		String sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".datacite_views as select * from datasetsusagestats_20210301.datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Datasets Views Table created");
+
+		logger.info("Creating Datasets Downloads Table");
+		sql = "Create TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
+			+ ".datacite_downloads as select * from datasetsusagestats_20210301.datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Datasets Downloads Table created");
+
+	}
+}

dhp-workflows/dhp-usage-raw-data-update/src/main/java/eu/dnetlib/oa/graph/usagerawdata/export/UsageStatsExporter.java

@@ -142,8 +142,20 @@ public class UsageStatsExporter {
 			sarcStats.updateSarcLogs();
 		}
 		logger.info("Sarc done");
-		// finalize usagestats
 
+		PiwikDownloadLogs_B2SHARE b2sharePiwikID = new PiwikDownloadLogs_B2SHARE(ExecuteWorkflow.matomoBaseURL,
+			ExecuteWorkflow.matomoAuthToken);
+		b2sharePiwikID.GetOpenAIREB2SHARELogs(ExecuteWorkflow.repoLogPath);
+		logger.info("B2SHARE done");
+
+		PiwikStatsDB_B2SHARE piwikstatsB2SHAREdb = new PiwikStatsDB_B2SHARE(ExecuteWorkflow.repoLogPath,
+			ExecuteWorkflow.portalLogPath);
+		piwikstatsB2SHAREdb.setCounterRobotsURL(cRobotsUrl);
+
+		logger.info("Processing B2SHARE logs");
+		piwikstatsB2SHAREdb.processB2SHARELogs();
+
+		// finalize usagestats
 		logger.info("Dropping tmp tables");
 		if (ExecuteWorkflow.finalizeStats) {
 			piwikstatsdb.finalizeStats();
@@ -161,6 +173,7 @@ public class UsageStatsExporter {
 		piwikstatsdb.recreateDBAndTables();
 
 		piwikstatsdb.createPedocsOldUsageData();
+
 		Statement stmt = ConnectDB.getHiveConnection().createStatement();
 
 		logger.info("Creating LaReferencia tables");

dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/export/usagerawdata_parameters.json

@@ -215,5 +215,11 @@
     "paramLongName": "numberOfDownloadThreads",
     "paramDescription": "Number of download threads",
     "paramRequired": true
+  },
+    {
+    "paramName": "b2shareID",
+    "paramLongName": "b2shareID",
+    "paramDescription": "B2SHARE Matomo ID",
+    "paramRequired": true
   }
 ]

dhp-workflows/dhp-usage-raw-data-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagerawdata/oozie_app/workflow.xml

@@ -1,4 +1,4 @@
-<workflow-app name="Usage Graph Stats" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="Usage Activity Raw Data" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>hiveMetastoreUris</name>
@@ -78,6 +78,7 @@
             <arg>--sarcNumberOfIssnToDownload</arg><arg>${sarcNumberOfIssnToDownload}</arg>
             <arg>--finalizeStats</arg><arg>${finalizeStats}</arg>
             <arg>--numberOfDownloadThreads</arg><arg>${numberOfDownloadThreads}</arg>
+            <arg>--b2shareID</arg><arg>${b2shareID}</arg>
             <capture-output/>
         </java>
         <ok to="End" />

dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/ConnectDB.java

@@ -82,7 +82,7 @@ public abstract class ConnectDB {
 		Date today = Calendar.getInstance().getTime();
 		String todayAsString = df.format(today);
 
-		return ConnectDB.usageStatsDBSchema + "_" + todayAsString;
+		return ConnectDB.usageStatsDBSchema + todayAsString;
 	}
 
 	public static String getStatsDBSchema() {

dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/PiwikStatsDB.java

@@ -35,20 +35,20 @@ public class PiwikStatsDB {
 
 	private void createDatabase() throws Exception {
 
-//		try {
-//
-//			stmt = ConnectDB.getHiveConnection().createStatement();
-//
-//			logger.info("Dropping usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
-//			String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + " CASCADE";
-//			stmt.executeUpdate(dropDatabase);
-//		} catch (Exception e) {
-//			logger.error("Failed to drop database: " + e);
-//			throw new Exception("Failed to drop database: " + e.toString(), e);
-//		}
-//
 		try {
+
 			stmt = ConnectDB.getHiveConnection().createStatement();
+
+			logger.info("Dropping usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
+			String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + " CASCADE";
+			stmt.executeUpdate(dropDatabase);
+		} catch (Exception e) {
+			logger.error("Failed to drop database: " + e);
+			throw new Exception("Failed to drop database: " + e.toString(), e);
+		}
+
+		try {
+
 			logger.info("Creating usagestats DB: " + ConnectDB.getUsageStatsDBSchema());
 			String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema();
 			stmt.executeUpdate(createDatabase);
@@ -132,7 +132,7 @@ public class PiwikStatsDB {
 			+ "max(views) AS count, max(openaire_referrer) AS openaire "
 			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_views_monthly_tmp p, "
 			+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
-			+ "WHERE p.source=d.piwik_id AND p.id=ro.oid AND ro.oid!='200' "
+			+ "WHERE p.source=d.piwik_id AND p.id=ro.oid AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
 			+ "GROUP BY d.id, ro.id, month "
 			+ "ORDER BY d.id, ro.id, month ";
 		stmt.executeUpdate(create_views_stats);
@@ -145,7 +145,7 @@ public class PiwikStatsDB {
 			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_views_monthly_tmp p, "
 			+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
 			+ "WHERE p.source=" + ExecuteWorkflow.portalMatomoID
-			+ " AND p.source=d.piwik_id and p.id=ro.id AND ro.oid!='200' "
+			+ " AND p.source=d.piwik_id and p.id=ro.id AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
 			+ "GROUP BY d.id, ro.id, month "
 			+ "ORDER BY d.id, ro.id, month ";
 		stmt.executeUpdate(create_pageviews_stats);
@@ -194,7 +194,7 @@ public class PiwikStatsDB {
 			+ "max(downloads) AS count, max(openaire_referrer) AS openaire "
 			+ "FROM " + ConnectDB.getUsageStatsDBSchema() + ".openaire_result_downloads_monthly_tmp p, "
 			+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
-			+ "WHERE p.source=d.piwik_id and p.id=ro.oid AND ro.oid!='200' "
+			+ "WHERE p.source=d.piwik_id and p.id=ro.oid AND ro.oid!='200' AND d.id!='re3data_____::7b0ad08687b2c960d5aeef06f811d5e6' "
 			+ "GROUP BY d.id, ro.id, month "
 			+ "ORDER BY d.id, ro.id, month ";
 		stmt.executeUpdate(sql);
@@ -337,6 +337,96 @@ public class PiwikStatsDB {
 
 	}
 
+	public void uploadB2SHAREStats() throws Exception {
+		stmt = ConnectDB.getHiveConnection().createStatement();
+		ConnectDB.getHiveConnection().setAutoCommit(false);
+
+		// Dropping B2SHARE b2share_result_views_monthly_tmp view
+		logger.info("Dropping B2SHARE b2share_result_views_monthly_tmp view");
+		String sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp";
+		logger.info("Dropped b2share_result_views_monthly_tmp view ");
+		stmt.executeUpdate(sql);
+
+		// Dropping B2SHARE b2share_result_views_monthly_tmp view
+		logger.info("Dropping b2SHARE b2share_result_downloads_monthly_tmp view");
+		sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp";
+		logger.info("Dropped b2share_result_downloads_monthly_tmp view ");
+		stmt.executeUpdate(sql);
+
+		// Dropping B2SHARE b2share_views_stats_tmp table
+		logger.info("Dropping B2SHARE b2share_views_stats_tmp table");
+		sql = "DROP TABLE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp";
+		logger.info("Dropped b2share_views_stats_tmp table ");
+		stmt.executeUpdate(sql);
+
+		// Dropping B2SHARE b2share_downloads_stats_tmp table
+		logger.info("Dropping B2SHARE b2share_downloads_stats_tmp table");
+		sql = "DROP TABLE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp";
+		logger.info("Dropped b2share_downloads_stats_tmp table ");
+		stmt.executeUpdate(sql);
+
+		// Creating B2SHARE b2share_result_views_monthly_tmp view
+		logger.info("Creating B2SHARE b2share_result_views_monthly_tmp view");
+		sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp "
+			+ "AS SELECT entity_id, reflect('java.net.URLDecoder', 'decode', entity_id) AS id, "
+			+ "COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, "
+			+ "CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source "
+			+ "FROM " + ConnectDB.getUsageRawDataDBSchema() + ".piwiklog "
+			+ "WHERE action='action' and (source_item_type='oaItem' or source_item_type='repItem') and source=412 "
+			+ "GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), source ORDER BY source, entity_id";
+		stmt.executeUpdate(sql);
+		logger.info("Created b2share_result_views_monthly_tmp view ");
+
+		// Creating B2SHARE b2share_views_stats_tmp table
+		logger.info("Creating B2SHARE b2share_views_stats_tmp table");
+		sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp AS "
+			+ "SELECT 'B2SHARE' as source, d.id as repository_id, ro.id as result_id, month as date, "
+			+ "max(views) AS count, max(openaire_referrer) AS openaire FROM " + ConnectDB.getUsageStatsDBSchema()
+			+ ".b2share_result_views_monthly_tmp p, "
+			+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
+			+ "WHERE p.id=ro.oid and d.id='re3data_____::ad3609c351bd520edf6f10f5e0d9b877' "
+			+ "GROUP BY d.id, ro.id, month ORDER BY d.id, ro.id";
+		stmt.executeUpdate(sql);
+		logger.info("Created B2SHARE b2share_views_stats_tmp table");
+
+		// Creating B2SHARE b2share_result_downloads_monthly_tmp view
+		logger.info("Creating B2SHARE b2share_result_downloads_monthly_tmp view");
+		sql = "CREATE OR REPLACE VIEW " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp "
+			+ "AS SELECT entity_id, reflect('java.net.URLDecoder', 'decode', entity_id) AS id, "
+			+ "COUNT(entity_id) as views, SUM(CASE WHEN referrer_name LIKE '%openaire%' THEN 1 ELSE 0 END) AS openaire_referrer, "
+			+ "CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')) AS month, source "
+			+ "FROM " + ConnectDB.getUsageRawDataDBSchema() + ".piwiklog "
+			+ "WHERE action='download' and (source_item_type='oaItem' or source_item_type='repItem') and source=412 "
+			+ "GROUP BY entity_id, CONCAT(YEAR(timestamp), '/', LPAD(MONTH(timestamp), 2, '0')), source ORDER BY source, entity_id";
+		stmt.executeUpdate(sql);
+		logger.info("Created b2share_result_downloads_monthly_tmp view ");
+
+		// Creating B2SHARE b2share_downloads_stats_tmp table
+		logger.info("Creating B2SHARE b2share_downloads_stats_tmp table");
+		sql = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp AS "
+			+ "SELECT 'B2SHARE' as source, d.id as repository_id, ro.id as result_id, month as date, "
+			+ "max(views) AS count, max(openaire_referrer) AS openaire FROM " + ConnectDB.getUsageStatsDBSchema()
+			+ ".b2share_result_downloads_monthly_tmp p, "
+			+ ConnectDB.getStatsDBSchema() + ".datasource d, " + ConnectDB.getStatsDBSchema() + ".result_oids ro "
+			+ "WHERE p.id=ro.oid and d.id='re3data_____::ad3609c351bd520edf6f10f5e0d9b877' "
+			+ "GROUP BY d.id, ro.id, month ORDER BY d.id, ro.id";
+		stmt.executeUpdate(sql);
+		logger.info("Created B2SHARE b2share_downloads_stats_tmp table");
+
+		// Dropping B2SHARE b2share_result_views_monthly_tmp view
+		logger.info("Dropping B2SHARE b2share_result_views_monthly_tmp view");
+		sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_views_monthly_tmp";
+		logger.info("Dropped b2share_result_views_monthly_tmp view ");
+		stmt.executeUpdate(sql);
+
+		// Dropping B2SHARE b2share_result_views_monthly_tmp view
+		logger.info("Dropping B2SHARE b2share_result_downloads_monthly_tmp view");
+		sql = "DROP view IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".b2share_result_downloads_monthly_tmp";
+		logger.info("Dropped b2share_result_downloads_monthly_tmp view ");
+		stmt.executeUpdate(sql);
+
+	}
+
 	public void finalizeStats() throws Exception {
 		stmt = ConnectDB.getHiveConnection().createStatement();
 		ConnectDB.getHiveConnection().setAutoCommit(false);
@@ -402,6 +492,13 @@ public class PiwikStatsDB {
 		stmt.executeUpdate(sql);
 		logger.info("LaReferencia views updated to views_stats");
 
+		// Inserting B2SHARE views stats
+		logger.info("Inserting B2SHARE data to views_stats");
+		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".views_stats "
+			+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".b2share_views_stats_tmp";
+		stmt.executeUpdate(sql);
+		logger.info("B2SHARE views updated to views_stats");
+
 		logger.info("Creating downloads_stats table");
 		String createDownloadsStats = "CREATE TABLE IF NOT EXISTS "
 			+ ConnectDB.getUsageStatsDBSchema()
@@ -425,12 +522,18 @@ public class PiwikStatsDB {
 		logger.info("Inserted Pedocs data to downloads_stats");
 
 		// Inserting TUDELFT downloads stats
-		logger.info("Inserting TUDELFT old data to downloads_stats");
+		logger.info("Inserting TUDELFT data to downloads_stats");
 		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
 			+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".tudelft_downloads_stats_tmp";
 		stmt.executeUpdate(sql);
 		logger.info("Inserted TUDELFT data to downloads_stats");
 
+		// Inserting B2SHARE downloads stats
+		logger.info("Inserting B2SHARE  data to downloads_stats");
+		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
+			+ "SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".b2share_downloads_stats_tmp";
+		stmt.executeUpdate(sql);
+		logger.info("Inserted B2SHARE data to downloads_stats");
 		// Inserting Lareferencia downloads stats
 		logger.info("Inserting LaReferencia data to downloads_stats");
 		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
@@ -452,6 +555,20 @@ public class PiwikStatsDB {
 		stmt.executeUpdate(sql);
 		logger.info("SARC-OJS downloads updated to downloads_stats");
 
+		// Inserting Datacite views stats
+		logger.info("Inserting Datacite views to views_stats");
+		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".views_stats "
+			+ "SELECT * FROM " + ConnectDB.getUsageRawDataDBSchema() + ".datacite_views";
+		stmt.executeUpdate(sql);
+		logger.info("Datacite views updated to views_stats");
+
+		// Inserting Datacite downloads stats
+		logger.info("Inserting Datacite downloads to downloads_stats");
+		sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats "
+			+ "SELECT * FROM " + ConnectDB.getUsageRawDataDBSchema() + ".datacite_downloads";
+		stmt.executeUpdate(sql);
+		logger.info("Datacite downloads updated to downloads_stats");
+
 		logger.info("Creating pageviews_stats table");
 		String create_pageviews_stats = "CREATE TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema()
 			+ ".pageviews_stats "

dhp-workflows/dhp-usage-stats-build/src/main/java/eu/dnetlib/oa/graph/usagestatsbuild/export/UsageStatsExporter.java

@@ -51,6 +51,9 @@ public class UsageStatsExporter {
 			logger.info("Processing TUDELFT Stats");
 			piwikstatsdb.uploadTUDELFTStats();
 			logger.info("Processing TUDELFT Stats Done");
+			logger.info("Processing B2SHARE Stats");
+			piwikstatsdb.uploadB2SHAREStats();
+			logger.info("Processing B2SHARE Stats Done");
 
 		}
 

dhp-workflows/dhp-usage-stats-build/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestatsbuild/oozie_app/workflow.xml

@@ -1,4 +1,4 @@
-<workflow-app name="Usage Graph Stats" xmlns="uri:oozie:workflow:0.5">
+<workflow-app name="Usage Stats Update" xmlns="uri:oozie:workflow:0.5">
     <parameters>
         <property>
             <name>hiveMetastoreUris</name>

nbactions.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>test</actionName>
+            <packagings>
+                <packaging>*</packaging>
+            </packagings>
+            <goals>
+                <goal>test</goal>
+            </goals>
+            <properties>
+                <skipTests>true</skipTests>
+            </properties>
+        </action>
+    </actions>