dhp-common/src/test/resources/eu/dnetlib/dhp/transform/terms.txt

@@ -1009,7 +1009,7 @@ datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13
 datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN
-datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN
+datacite:id_typologies @=@ datacite:id_typologies @=@ issn @=@ issn
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC
 datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/input_itgv4.xml

@@ -31,7 +31,7 @@
                     <datacite:relatedIdentifier relatedIdentifierType="URL" relationType="HasMetadata">https://pub.uni-bielefeld.de/record/1997560.json</datacite:relatedIdentifier>
                 </datacite:relatedIdentifiers>
                 <datacite:alternateIdentifiers>
-                    <datacite:alternateIdentifier alternateIdentifierType="ISSN">0016-9056</datacite:alternateIdentifier>
+                    <datacite:alternateIdentifier alternateIdentifierType="issn">0016-9056</datacite:alternateIdentifier>
                 </datacite:alternateIdentifiers>
                 <dc:language>ger</dc:language>
                 <dc:publisher>Friedrich</dc:publisher>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/original/dc_cleaning_OPENAIREplus_compliant_hal_orig

@@ -134,9 +134,9 @@ oaf:identifier = set(xpath:"//dri:recordIdentifier", @identifierType = "oai-orig
 oaf:datasourceprefix = xpath:"//oaf:datasourceprefix";
 
 // journal data
-// avoiding regular expressions, while a) correcting ISSNs with no - or other letters instead of - and b) ignoring any stuff after the ISSN (as e.g. print/online/...)
+// avoiding regular expressions, while a) correcting ISSNs with no - or other letters instead of - and b) ignoring any stuff after the issn (as e.g. print/online/...)
-$varISSN = xpath:"//dc:source[starts-with(., 'ISSN:') and string-length(.) > 12]/concat(substring(normalize-space(substring-after(., 'ISSN:')), 1, 4), '-', normalize-space(substring-after(., substring(normalize-space(substring-after(., 'ISSN:')), 1, 4))))";
+$varISSN = xpath:"//dc:source[starts-with(., 'issn:') and string-length(.) > 12]/concat(substring(normalize-space(substring-after(., 'issn:')), 1, 4), '-', normalize-space(substring-after(., substring(normalize-space(substring-after(., 'issn:')), 1, 4))))";
-//$varEISSN = xpath:"//dc:source[starts-with(., 'EISSN:') and string-length(.) > 13]/normalize-space(substring-after(., 'ISSN:'))";
+//$varEISSN = xpath:"//dc:source[starts-with(., 'EISSN:') and string-length(.) > 13]/normalize-space(substring-after(., 'issn:'))";
 $varEISSN = xpath:"//dc:source[starts-with(., 'EISSN:') and string-length(.) > 13]/concat(substring(normalize-space(substring-after(., 'EISSN:')), 1, 4), '-', normalize-space(substring-after(., substring(normalize-space(substring-after(., 'EISSN:')), 1, 4))))";
 oaf:journal = set(xpath:"//oaf:datasourceprefix[$varISSN or $varEISSN]/''", @issn = xpath:"$varISSN";, @eissn = xpath:"$varEISSN";);
 

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/original/xslt_cleaning_oaiOpenaire_datacite_ExchangeLandingpagePid_orig.xsl

@@ -169,7 +169,7 @@
 
 
                         <xsl:for-each select="(//datacite:alternateIdentifier, //datacite:identifier)
-                                       [not($varCobjCategory = '0001' and ./@alternateIdentifierType = ('ISSN', 'EISSN'))]
+                                       [not($varCobjCategory = '0001' and ./@alternateIdentifierType = ('issn', 'EISSN'))]
                                        [not($varCobjCategory = '0013' and ./@alternateIdentifierType = 'ISBN')]
                                        [not(//oaf:datasourceprefix = 'od______4225' and ends-with(., 'pdf'))]
                                        [not(//oaf:datasourceprefix = ('od______1562', 'od______4732'))]
@@ -182,7 +182,7 @@
                                 <xsl:value-of select="."/>
                             </datacite:alternateIdentifier>
                             <!--
-                                    <xsl:copy-of select="//datacite:alternateIdentifier[not($varCobjCategory = '0001' and ./@alternateIdentifierType = 'ISSN')][not(//oaf:datasourceprefix = 'od______4225' and ends-with(., 'pdf'))]"  copy-namespaces="no"/>
+                                    <xsl:copy-of select="//datacite:alternateIdentifier[not($varCobjCategory = '0001' and ./@alternateIdentifierType = 'issn')][not(//oaf:datasourceprefix = 'od______4225' and ends-with(., 'pdf'))]"  copy-namespaces="no"/>
                                     <xsl:copy-of select="//datacite:identifier[not(//oaf:datasourceprefix = 'od______4225' and ends-with(., 'pdf'))]"  copy-namespaces="no"/>
                             -->
                         </xsl:for-each>
@@ -283,7 +283,7 @@
                     </datacite:alternateIdentifiers>
 
                     <!--
-                                                           [$varCobjCategory = '0001' and ./@alternateIdentifierType = 'ISSN'][1]">
+                                                           [$varCobjCategory = '0001' and ./@alternateIdentifierType = 'issn'][1]">
                     -->
                     <datacite:relatedIdentifiers>
                         <!--
@@ -300,7 +300,7 @@
                         <xsl:copy-of select="//datacite:relatedIdentifier"  copy-namespaces="no"/>
 
                         <xsl:for-each select="(//datacite:alternateIdentifier, //datacite:identifier)
-                                       [$varCobjCategory = '0001' and ./@alternateIdentifierType = ('ISSN', 'EISSN')]">
+                                       [$varCobjCategory = '0001' and ./@alternateIdentifierType = ('issn', 'EISSN')]">
                             <datacite:relatedIdentifier>
                                 <xsl:attribute name="relatedIdentifierType" select="./@alternateIdentifierType"/>
                                 <xsl:attribute name="relationType" select="'isPartOf'"/>
@@ -770,13 +770,13 @@
                 </oaf:collectedFrom>
 
                 <!-- oaf:journal -->
-                <!-- ISSN  is erroneously stuffed in alternateIdentifier, should be put into relatedIdentifier -->
+                <!-- issn  is erroneously stuffed in alternateIdentifier, should be put into relatedIdentifier -->
                 <!-- check/ensure that oaf:journal is not prepared for a journal itself, perhaps limit to Rothamsted -->
                 <!-- Huelva marks L, E ISSNs as ISSNs, with mark within field in spaces or after blanc -->
                 <!-- Qeios declares many records as text, although many seem to be definitions which are also related to 'journal' volumes/issues -->
-                <xsl:if test="($varCobjCategory = '0001' or contains(//dri:recordIdentifier, 'www.qeios.com')) and (//*[local-name() = 'alternateIdentifier']/@alternateIdentifierType[. = 'ISSN'] or //*[local-name() = 'relatedIdentifier'][lower-case(@relationType) = 'ispartof'][@relatedIdentifierType = 'ISSN'])">
+                <xsl:if test="($varCobjCategory = '0001' or contains(//dri:recordIdentifier, 'www.qeios.com')) and (//*[local-name() = 'alternateIdentifier']/@alternateIdentifierType[. = 'issn'] or //*[local-name() = 'relatedIdentifier'][lower-case(@relationType) = 'ispartof'][@relatedIdentifierType = 'issn'])">
                     <oaf:journal>
-                        <xsl:attribute name="issn" select="(//*[local-name() = 'alternateIdentifier'][./@alternateIdentifierType = 'ISSN'], //*[local-name() = 'relatedIdentifier'][lower-case(./@relationType) = 'ispartof'][./@relatedIdentifierType = 'ISSN'])[1]/concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
+                        <xsl:attribute name="issn" select="(//*[local-name() = 'alternateIdentifier'][./@alternateIdentifierType = 'issn'], //*[local-name() = 'relatedIdentifier'][lower-case(./@relationType) = 'ispartof'][./@relatedIdentifierType = 'issn'])[1]/concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
                         <xsl:if test="//*[local-name() = 'citationVolume']">
                             <xsl:attribute name="vol" select="//*[local-name() = 'citationVolume']"/>
                         </xsl:if>
@@ -793,9 +793,9 @@
                     </oaf:journal>
                 </xsl:if>
 
-                <xsl:if test="$varCobjCategory = '0001' and //*[local-name() = 'relatedIdentifier']/@relatedIdentifierType[. = ('ISSN', 'PISSN', 'EISSN', 'LISSN')]">
+                <xsl:if test="$varCobjCategory = '0001' and //*[local-name() = 'relatedIdentifier']/@relatedIdentifierType[. = ('issn', 'PISSN', 'EISSN', 'LISSN')]">
                     <oaf:journal>
-                        <xsl:for-each select="//*[local-name() = 'relatedIdentifier'][@relatedIdentifierType[. = ('ISSN', 'PISSN', 'EISSN', 'LISSN')]][@relationType/lower-case(.) = 'ispartof']">
+                        <xsl:for-each select="//*[local-name() = 'relatedIdentifier'][@relatedIdentifierType[. = ('issn', 'PISSN', 'EISSN', 'LISSN')]][@relationType/lower-case(.) = 'ispartof']">
 
                             <xsl:attribute name="{./@relatedIdentifierType}" select="./concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
                             <!-- -->
@@ -844,7 +844,7 @@
     <!--
       <xsl:template match="//*[local-name() = 'resource']/*[local-name()='alternateIdentifiers']">
         <xsl:copy copy-namespaces="no">
-         <xsl:copy-of select="./*[not($varCobjCategory = '0001' and ./@alternateIdentifierType = 'ISSN')]"  copy-namespaces="no"/>
+         <xsl:copy-of select="./*[not($varCobjCategory = '0001' and ./@alternateIdentifierType = 'issn')]"  copy-namespaces="no"/>
 
         <xsl:if test="//*[local-name() = 'resource']/*[local-name()='identifier'][lower-case(@identifierType)='handle']">
               <datacite:alternateIdentifier>

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/xslt_cleaning_oaiOpenaire_datacite_ExchangeLandingpagePid.xsl

@@ -159,7 +159,7 @@
 
 
                         <xsl:for-each select="(//datacite:alternateIdentifier, //datacite:identifier)
-                                       [not($varCobjCategory = '0001' and ./@alternateIdentifierType = ('ISSN', 'EISSN'))]
+                                       [not($varCobjCategory = '0001' and ./@alternateIdentifierType = ('issn', 'EISSN'))]
                                        [not($varCobjCategory = '0013' and ./@alternateIdentifierType = 'ISBN')]
                                        [not(//oaf:datasourceprefix = 'od______4225' and ends-with(., 'pdf'))]
                                        [not(//oaf:datasourceprefix = ('od______1562', 'od______4732'))]
@@ -212,7 +212,7 @@
                         <xsl:copy-of select="//datacite:relatedIdentifier"  copy-namespaces="no"/>
 
                         <xsl:for-each select="(//datacite:alternateIdentifier, //datacite:identifier)
-                                       [$varCobjCategory = '0001' and ./@alternateIdentifierType = ('ISSN', 'EISSN')]">
+                                       [$varCobjCategory = '0001' and ./@alternateIdentifierType = ('issn', 'EISSN')]">
                             <datacite:relatedIdentifier>
                                 <xsl:attribute name="relatedIdentifierType" select="./@alternateIdentifierType"/>
                                 <xsl:attribute name="relationType" select="'isPartOf'"/>
@@ -571,13 +571,13 @@
                 </oaf:collectedFrom>
 
                 <!-- oaf:journal -->
-                <!-- ISSN  is erroneously stuffed in alternateIdentifier, should be put into relatedIdentifier -->
+                <!-- issn  is erroneously stuffed in alternateIdentifier, should be put into relatedIdentifier -->
                 <!-- check/ensure that oaf:journal is not prepared for a journal itself, perhaps limit to Rothamsted -->
                 <!-- Huelva marks L, E ISSNs as ISSNs, with mark within field in spaces or after blanc -->
                 <!-- Qeios declares many records as text, although many seem to be definitions which are also related to 'journal' volumes/issues -->
-                <xsl:if test="($varCobjCategory = '0001' or contains(//dri:recordIdentifier, 'www.qeios.com')) and (//*[local-name() = 'alternateIdentifier']/@alternateIdentifierType[. = 'ISSN'] or //*[local-name() = 'relatedIdentifier'][lower-case(@relationType) = 'ispartof'][@relatedIdentifierType = 'ISSN'])">
+                <xsl:if test="($varCobjCategory = '0001' or contains(//dri:recordIdentifier, 'www.qeios.com')) and (//*[local-name() = 'alternateIdentifier']/@alternateIdentifierType[. = 'issn'] or //*[local-name() = 'relatedIdentifier'][lower-case(@relationType) = 'ispartof'][@relatedIdentifierType = 'issn'])">
                     <oaf:journal>
-                        <xsl:attribute name="issn" select="(//*[local-name() = 'alternateIdentifier'][./@alternateIdentifierType = 'ISSN'], //*[local-name() = 'relatedIdentifier'][lower-case(./@relationType) = 'ispartof'][./@relatedIdentifierType = 'ISSN'])[1]/concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
+                        <xsl:attribute name="issn" select="(//*[local-name() = 'alternateIdentifier'][./@alternateIdentifierType = 'issn'], //*[local-name() = 'relatedIdentifier'][lower-case(./@relationType) = 'ispartof'][./@relatedIdentifierType = 'issn'])[1]/concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
                         <xsl:if test="//*[local-name() = 'citationVolume']">
                             <xsl:attribute name="vol" select="//*[local-name() = 'citationVolume']"/>
                         </xsl:if>
@@ -594,9 +594,9 @@
                     </oaf:journal>
                 </xsl:if>
 
-                <xsl:if test="$varCobjCategory = '0001' and //*[local-name() = 'relatedIdentifier']/@relatedIdentifierType[. = ('ISSN', 'PISSN', 'EISSN', 'LISSN')]">
+                <xsl:if test="$varCobjCategory = '0001' and //*[local-name() = 'relatedIdentifier']/@relatedIdentifierType[. = ('issn', 'PISSN', 'EISSN', 'LISSN')]">
                     <oaf:journal>
-                        <xsl:for-each select="//*[local-name() = 'relatedIdentifier'][@relatedIdentifierType[. = ('ISSN', 'PISSN', 'EISSN', 'LISSN')]][@relationType/lower-case(.) = 'ispartof']">
+                        <xsl:for-each select="//*[local-name() = 'relatedIdentifier'][@relatedIdentifierType[. = ('issn', 'PISSN', 'EISSN', 'LISSN')]][@relationType/lower-case(.) = 'ispartof']">
 
                             <xsl:attribute name="{./@relatedIdentifierType}" select="./concat(substring(., 1, 4), '-', substring(., string-length(.)-3, 4))"/>
                             <!-- -->

dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/terms.txt

@@ -1009,7 +1009,7 @@ datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13
 datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN
-datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN
+datacite:id_typologies @=@ datacite:id_typologies @=@ issn @=@ issn
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC
 datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID

dhp-workflows/dhp-doiboost/src/main/resources/eu.dnetlib.dhp.doiboost.mappings/crossref_mapping.csv

@@ -36,10 +36,10 @@ article-number,String,No,,N/A,
 published-print,Partial Date,No,Date on which the work was published in print,"Result/relevantDate with Qualifier(""published-print"", ""dnet:dataCite_date"")",
 published-online,Partial Date,No,Date on which the work was published online,"Result/relevantDate with Qualifier(""published-online"", ""dnet:dataCite_date"")",
 subject,Array of String,No,"Subject category names, a controlled vocabulary from Sci-Val. Available for most journal articles","Result/subject with Qualifier(""keywords"", ""dnet:subject_classification_typologies""). ","Future improvements: map the controlled vocabulary instead of using the generic ""keywords"" qualifier"
-ISSN,Array of String,No,,"Publication/Journal/issn
+issn,Array of String,No,,"Publication/Journal/issn
 Publication/Journal/lissn
 Publication/Journal/eissn",The mapping depends on the value of issn-type
-issn-type,Array of ISSN with Type,No,List of ISSNs with ISSN type information,N/A,Its value guides the setting of the properties in Journal (see row above)
+issn-type,Array of issn with Type,No,List of ISSNs with issn type information,N/A,Its value guides the setting of the properties in Journal (see row above)
 ISBN,Array of String,No,,Publication/source,"In case of Book We can map ISBN and container title on Publication/source using this syntax container-title + ""ISBN: "" + ISBN"
 archive,Array of String,No,,N/A,
 license,Array of License,No,,Result/Instance/License,

dhp-workflows/dhp-doiboost/src/test/resources/eu/dnetlib/dhp/doiboost/issn_gold_oa_version_4.csv

@@ -1,4 +1,4 @@
-"ISSN","ISSN_L","ISSN_IN_DOAJ","ISSN_IN_ROAD","ISSN_IN_PMC","ISSN_IN_OAPC","ISSN_IN_WOS","ISSN_IN_SCOPUS","JOURNAL_IN_DOAJ","JOURNAL_IN_ROAD","JOURNAL_IN_PMC","JOURNAL_IN_OAPC","JOURNAL_IN_WOS","JOURNAL_IN_SCOPUS","TITLE","TITLE_SOURCE"
+"issn","ISSN_L","ISSN_IN_DOAJ","ISSN_IN_ROAD","ISSN_IN_PMC","ISSN_IN_OAPC","ISSN_IN_WOS","ISSN_IN_SCOPUS","JOURNAL_IN_DOAJ","JOURNAL_IN_ROAD","JOURNAL_IN_PMC","JOURNAL_IN_OAPC","JOURNAL_IN_WOS","JOURNAL_IN_SCOPUS","TITLE","TITLE_SOURCE"
 "0001-625X","0001-625X",1,1,0,0,0,1,1,1,0,0,0,1,"Acta Mycologica","DOAJ"
 "0002-0397","0002-0397",1,1,0,0,1,1,1,1,0,0,1,1,"Africa Spectrum","DOAJ"
 "0003-2565","0003-2565",1,0,0,0,0,0,1,0,0,0,0,0,"Anali Pravnog Fakulteta u Beogradu","DOAJ"

dhp-workflows/dhp-graph-mapper/pom.xml

@@ -122,6 +122,11 @@
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_2.11</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.opencsv</groupId>
+            <artifactId>opencsv</artifactId>
+            <version>5.5</version>
+        </dependency>
 
     </dependencies>
 

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/Aggregators.scala

@@ -0,0 +1,54 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap
+
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, TypedColumn}
+import org.apache.spark.sql.expressions.Aggregator
+
+
+case class HostedByItemType(id: String, officialname: String, issn: String, eissn: String, lissn: String, openAccess: Boolean) {}
+
+
+object Aggregators {
+
+
+
+  def getId(s1:String, s2:String) : String = {
+    if (!s1.equals("")){
+      return s1}
+    s2
+  }
+
+
+  def createHostedByItemTypes(df: Dataset[HostedByItemType]): Dataset[HostedByItemType] = {
+      val transformedData : Dataset[HostedByItemType] = df
+          .groupByKey(_.id)(Encoders.STRING)
+        .agg(Aggregators.hostedByAggregator)
+          .map{
+            case (id:String , res:HostedByItemType) => res
+          }(Encoders.product[HostedByItemType])
+
+    transformedData
+    }
+
+  val hostedByAggregator: TypedColumn[HostedByItemType, HostedByItemType] = new Aggregator[HostedByItemType, HostedByItemType, HostedByItemType] {
+    override def zero: HostedByItemType = HostedByItemType("","","","","",false)
+    override def reduce(b: HostedByItemType, a:HostedByItemType): HostedByItemType = {
+      return merge(b, a)
+    }
+    override def merge(b1: HostedByItemType, b2: HostedByItemType): HostedByItemType = {
+      if (b1 == null){
+        return b2
+      }
+      if(b2 == null){
+        return b1
+      }
+
+      HostedByItemType(getId(b1.id, b2.id), getId(b1.officialname, b2.officialname),  getId(b1.issn, b2.issn), getId(b1.eissn, b2.eissn), getId(b1.lissn, b2.lissn), b1.openAccess || b2.openAccess)
+
+    }
+    override def finish(reduction: HostedByItemType): HostedByItemType = reduction
+    override def bufferEncoder: Encoder[HostedByItemType] = Encoders.product[HostedByItemType]
+
+    override def outputEncoder: Encoder[HostedByItemType] = Encoders.product[HostedByItemType]
+  }.toColumn
+
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/Constants.java

@@ -0,0 +1,15 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap;
+
+public class Constants {
+
+
+
+    public static final String OPENAIRE = "openaire";
+    public static final String DOAJ = "doaj";
+    public static final String UNIBI = "unibi";
+
+
+    public static final String ISSN = "issn";
+    public static final String EISSN = "eissn";
+    public static final String ISSNL = "issnl";
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/GetCSV.java

@@ -0,0 +1,111 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.opencsv.bean.CsvToBeanBuilder;
+import eu.dnetlib.dhp.oa.graph.hostebymap.model.UnibiGoldModel;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.*;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+
+public class GetCSV {
+    private static final Log log = LogFactory.getLog(eu.dnetlib.dhp.oa.graph.hostebymap.GetCSV.class);
+
+    public static void main(final String[] args) throws Exception {
+        final ArgumentApplicationParser parser = new ArgumentApplicationParser(
+                IOUtils
+                        .toString(
+                                GetCSV.class
+                                        .getResourceAsStream(
+                                                "/eu/dnetlib/dhp/oa/graph/hostedbymap/download_csv_parameters.json")));
+
+        parser.parseArgument(args);
+
+        final String fileURL = parser.get("fileURL");
+        final String hdfsPath = parser.get("hdfsPath");
+        final String hdfsNameNode = parser.get("hdfsNameNode");
+        final String classForName = parser.get("classForName");
+        final Boolean shouldReplace = Optional.ofNullable((parser.get("replace")))
+                .map(Boolean::valueOf)
+                .orElse(false);
+
+
+        URLConnection connection = new URL(fileURL).openConnection();
+        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
+        connection.connect();
+
+        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
+
+        if(shouldReplace){
+            PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ.csv")));
+            String line = null;
+            while((line = in.readLine())!= null){
+                writer.println(line.replace("\\\"", "\""));
+            }
+            writer.close();
+            in.close();
+            in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
+        }
+
+        Configuration conf = new Configuration();
+        conf.set("fs.defaultFS", hdfsNameNode);
+
+        FileSystem fileSystem = FileSystem.get(conf);
+        Path hdfsWritePath = new Path(hdfsPath);
+        FSDataOutputStream fsDataOutputStream = null;
+        if (fileSystem.exists(hdfsWritePath)) {
+            fileSystem.delete(hdfsWritePath, false);
+        }
+        fsDataOutputStream = fileSystem.create(hdfsWritePath);
+
+        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
+
+        Class<?> clazz = Class.forName(classForName);
+
+        ObjectMapper mapper = new ObjectMapper();
+
+        new CsvToBeanBuilder(in)
+                .withType(clazz)
+                .withMultilineLimit(1)
+                .build()
+                .parse()
+        .forEach(line -> {
+            try {
+                writer.write(mapper.writeValueAsString(line));
+                writer.newLine();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+
+
+        writer.close();
+        in.close();
+        if(shouldReplace){
+            File f = new File("/tmp/DOAJ.csv");
+            f.delete();
+        }
+
+
+    }
+
+
+
+
+}
+

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/SparkPrepareHostedByMapData.scala

@@ -0,0 +1,158 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap
+
+import eu.dnetlib.dhp.application.ArgumentApplicationParser
+import eu.dnetlib.dhp.oa.graph.hostebymap.model.{DOAJModel, UnibiGoldModel}
+import eu.dnetlib.dhp.oa.merge.AuthorMerger
+import eu.dnetlib.dhp.schema.common.ModelConstants
+import eu.dnetlib.dhp.schema.oaf.{Datasource, Organization, Publication, Relation}
+import org.apache.commons.io.IOUtils
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{Dataset, Encoder, Encoders, SaveMode, SparkSession}
+import org.json4s.DefaultFormats
+import org.slf4j.{Logger, LoggerFactory}
+import org.json4s.jackson.Serialization.write
+
+import scala.collection.mutable.ListBuffer
+
+object SparkPrepareHostedByMapData {
+
+  case class HostedByInfo(id: Option[String], officialname: String, journal_id: String, provenance : String, id_type: String) {}
+
+  implicit val tupleForJoinEncoder: Encoder[(String, HostedByItemType)] = Encoders.tuple(Encoders.STRING, Encoders.product[HostedByItemType])
+  implicit val mapEncoderDats: Encoder[Datasource] = Encoders.bean(classOf[Datasource])
+  implicit val mapEncoderDOAJ: Encoder[DOAJModel] = Encoders.kryo[DOAJModel]
+  implicit val mapEncoderUnibi: Encoder[UnibiGoldModel] = Encoders.kryo[UnibiGoldModel]
+  implicit val mapEncoderHBI: Encoder[HostedByInfo] = Encoders.product[HostedByInfo]
+
+
+  def toHostedByItemType(input: ((HostedByInfo, HostedByInfo), HostedByInfo)) : HostedByItemType = {
+    val openaire: HostedByInfo = input._1._1
+    val doaj: HostedByInfo = input._1._2
+    val gold: HostedByInfo = input._2
+    val isOpenAccess: Boolean = doaj == null && gold == null
+
+    openaire.journal_id match {
+      case Constants.ISSN => return HostedByItemType(openaire.id.get, openaire.officialname, openaire.journal_id, "", "", isOpenAccess)
+      case Constants.EISSN => return HostedByItemType(openaire.id.get, openaire.officialname, "", openaire.journal_id, "", isOpenAccess)
+      case Constants.ISSNL => return HostedByItemType(openaire.id.get, openaire.officialname, "", "", openaire.journal_id, isOpenAccess)
+
+      // catch the default with a variable so you can print it
+      case whoa => return null
+    }
+  }
+
+  def toHostedByMap(input: HostedByItemType): ListBuffer[String] = {
+    implicit val formats = DefaultFormats
+    val serializedJSON:String = write(input)
+
+    var hostedBy  = new ListBuffer[String]()
+    if(!input.issn.equals("")){
+      hostedBy += "{\"" + input.issn + "\":" + serializedJSON + "}"
+    }
+    if(!input.eissn.equals("")){
+      hostedBy += "{\"" + input.eissn + "\":" + serializedJSON + "}"
+    }
+    if(!input.lissn.equals("")){
+      hostedBy += "{\"" + input.lissn + "\":" + serializedJSON + "}"
+    }
+
+    hostedBy
+
+  }
+
+
+  def readOADataset(input:String, spark: SparkSession): Dataset[HostedByInfo] = {
+    spark.read.textFile(input).as[Datasource].flatMap(ds => {
+      val lst = new ListBuffer[HostedByInfo]()
+      if (ds.getJournal == null) {
+        return null
+      }
+      val issn: String = ds.getJournal.getIssnPrinted
+      val issnl: String = ds.getJournal.getIssnOnline
+      val eissn: String = ds.getJournal.getIssnOnline
+      val id: String = ds.getId
+      val officialname: String = ds.getOfficialname.getValue
+      if (issn != null) {
+        lst += HostedByInfo(Some(id), officialname, issn, Constants.OPENAIRE, Constants.ISSN)
+      }
+      if (issnl != null) {
+        lst += HostedByInfo(Some(id), officialname, issnl, Constants.OPENAIRE, Constants.ISSNL)
+      }
+      if (eissn != null) {
+        lst += HostedByInfo(Some(id), officialname, eissn, Constants.OPENAIRE, Constants.EISSN)
+      }
+      lst
+    }).filter(i => i != null)
+  }
+
+  def main(args: Array[String]): Unit = {
+
+    val logger: Logger = LoggerFactory.getLogger(getClass)
+    val conf: SparkConf = new SparkConf()
+    val parser = new ArgumentApplicationParser(IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/hostedby/prepare_hostedby_params.json")))
+    parser.parseArgument(args)
+    val spark: SparkSession =
+      SparkSession
+        .builder()
+        .config(conf)
+        .appName(getClass.getSimpleName)
+        .master(parser.get("master")).getOrCreate()
+
+    import spark.implicits._
+
+    val datasourcePath = parser.get("datasourcePath")
+    val workingDirPath = parser.get("workingPath")
+
+
+
+
+    logger.info("Getting the Datasources")
+
+    val doajDataset: Dataset[DOAJModel] = spark.read.load(workingDirPath + "/doaj").as[DOAJModel]
+    val unibiDataset: Dataset[UnibiGoldModel] = spark.read.load(datasourcePath).as[UnibiGoldModel]
+
+    val oa: Dataset[HostedByInfo] = readOADataset(datasourcePath, spark)
+
+    val doaj: Dataset[HostedByInfo] = doajDataset.flatMap(doaj => {
+      val lst = new ListBuffer[HostedByInfo]()
+      val issn: String = doaj.getIssn
+      val eissn: String = doaj.getEissn
+      val officialname: String = doaj.getJournalTitle
+      if (issn != null) {
+        lst += HostedByInfo(null, officialname, issn, Constants.DOAJ, Constants.ISSN)
+      }
+      if (eissn != null) {
+        lst += HostedByInfo(null, officialname, eissn, Constants.DOAJ, Constants.EISSN)
+      }
+      lst
+    })
+
+    val gold: Dataset[HostedByInfo] = unibiDataset.flatMap(gold => {
+      val lst = new ListBuffer[HostedByInfo]()
+      val issn: String = gold.getIssn
+      val issnl: String = gold.getIssn_l
+      val officialname: String = gold.getTitle
+      if (issn != null) {
+        lst += HostedByInfo(null, officialname, issn, Constants.UNIBI, Constants.ISSN)
+      }
+      if (issnl != null) {
+        lst += HostedByInfo(null, officialname, issnl, Constants.UNIBI, Constants.ISSNL)
+      }
+      lst
+    })
+
+    Aggregators.createHostedByItemTypes(oa.joinWith(doaj, oa.col("journal_id").equalTo(doaj.col("journal_id")), "left")
+      .joinWith(gold, $"_1.col('journal_id')".equalTo(gold.col("journal_id")), "left").map(toHostedByItemType)
+      .filter(i => i != null))
+      .flatMap(toHostedByMap)
+      //      .map(i => (i.id,i))
+      //      .groupByKey(_._1)
+      //      .agg(hostedByAggregator.toColumn)
+      //      .map(p => p._2)
+      .write.mode(SaveMode.Overwrite).save(s"$workingDirPath/HostedByMap")
+
+
+  }
+
+
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/model/DOAJModel.java

@@ -0,0 +1,53 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap.model;
+
+import java.io.Serializable;
+
+import com.opencsv.bean.CsvBindByName;
+
+
+public class DOAJModel implements Serializable {
+    @CsvBindByName(column = "Journal title")
+    private String journalTitle;
+
+    @CsvBindByName(column = "Journal ISSN (print version)")
+    private String issn ;
+
+    @CsvBindByName(column = "Journal EISSN (online version)")
+    private String eissn;
+
+    @CsvBindByName(column = "Review process")
+    private String reviewProcess;
+
+
+    public String getJournalTitle() {
+        return journalTitle;
+    }
+
+    public void setJournalTitle(String journalTitle) {
+        this.journalTitle = journalTitle;
+    }
+
+    public String getIssn() {
+        return issn;
+    }
+
+    public void setIssn(String issn) {
+        this.issn = issn;
+    }
+
+    public String getEissn() {
+        return eissn;
+    }
+
+    public void setEissn(String eissn) {
+        this.eissn = eissn;
+    }
+
+    public String getReviewProcess() {
+        return reviewProcess;
+    }
+
+    public void setReviewProcess(String reviewProcess) {
+        this.reviewProcess = reviewProcess;
+    }
+}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostebymap/model/UnibiGoldModel.java

@@ -0,0 +1,44 @@
+package eu.dnetlib.dhp.oa.graph.hostebymap.model;
+
+import com.opencsv.bean.CsvBindByName;
+
+import java.io.Serializable;
+
+public class UnibiGoldModel implements Serializable {
+    @CsvBindByName(column = "ISSN")
+    private String issn;
+    @CsvBindByName(column = "ISSN_L")
+    private String issn_l;
+    @CsvBindByName(column = "TITLE")
+    private String title;
+    @CsvBindByName(column = "TITLE_SOURCE")
+    private String title_source;
+
+    public String getIssn() {
+        return issn;
+    }
+
+    public void setIssn(String issn) {
+        this.issn = issn;
+    }
+
+    public String getIssn_l() {
+        return issn_l;
+    }
+
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public String getTitle_source() {
+        return title_source;
+    }
+
+    public void setTitle_source(String title_source) {
+        this.title_source = title_source;
+    }
+}

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestPreprocess.scala

@@ -0,0 +1,59 @@
+package eu.dnetlib.dhp.oa.graph.hostedbymap
+
+import java.sql.Timestamp
+
+import eu.dnetlib.dhp.oa.graph.hostebymap.SparkPrepareHostedByMapData
+import eu.dnetlib.dhp.oa.graph.hostebymap.SparkPrepareHostedByMapData.HostedByInfo
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{Dataset, SparkSession}
+import org.codehaus.jackson.map.ObjectMapper
+import org.json4s.DefaultFormats
+import org.junit.jupiter.api.Assertions.{assertNotNull, assertTrue}
+import org.junit.jupiter.api.Test
+import org.slf4j.{Logger, LoggerFactory}
+
+import scala.io.Source
+
+class TestPreprocess {
+
+  val logger: Logger = LoggerFactory.getLogger(getClass)
+  val mapper = new ObjectMapper()
+
+
+
+  @Test
+  def readDatasource():Unit = {
+
+
+    import org.apache.spark.sql.Encoders
+    implicit val formats = DefaultFormats
+    import org.json4s.jackson.Serialization.write
+
+    val conf = new SparkConf()
+    conf.setMaster("local[*]")
+    conf.set("spark.driver.host", "localhost")
+    val spark: SparkSession =
+      SparkSession
+        .builder()
+        .appName(getClass.getSimpleName)
+        .config(conf)
+        .getOrCreate()
+    val path = getClass.getResource("datasource.json").getPath
+
+
+    val schema = Encoders.product[HostedByInfo]
+
+    spark.read.textFile(path).foreach(r => println(mapper.writeValueAsString(r)))
+
+//    SparkPrepareHostedByMapData.readOADataset(path, spark)
+//      .foreach(r => println(write(r)))
+
+
+    spark.close()
+  }
+
+
+
+
+
+}

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java

@@ -0,0 +1,111 @@
+package eu.dnetlib.dhp.oa.graph.hostedbymap;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.opencsv.bean.CsvToBeanBuilder;
+import eu.dnetlib.dhp.oa.graph.hostebymap.GetCSV;
+import eu.dnetlib.dhp.oa.graph.hostebymap.model.UnibiGoldModel;
+import org.junit.jupiter.api.Test;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+public class TestReadCSV  {
+
+    @Test
+    public void testCSVUnibi() throws FileNotFoundException {
+
+
+        final String sourcePath = getClass()
+                .getResource("/eu/dnetlib/dhp/oa/graph/hostedbymap/unibiGold.csv")
+                .getPath();
+
+            List<UnibiGoldModel> beans = new CsvToBeanBuilder(new FileReader(sourcePath))
+                    .withType(UnibiGoldModel.class)
+                    .build()
+                    .parse();
+
+        ObjectMapper mapper = new ObjectMapper();
+
+            beans.forEach(r -> {
+                try {
+                    System.out.println(mapper.writeValueAsString(r));
+                } catch (JsonProcessingException e) {
+                    e.printStackTrace();
+                }
+            });
+
+
+    }
+
+    @Test
+    public void testCSVUrlUnibi() throws IOException {
+
+        URL csv = new URL("https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv");
+
+        BufferedReader in = new BufferedReader(new InputStreamReader(csv.openStream()));
+        ObjectMapper mapper = new ObjectMapper();
+
+        new CsvToBeanBuilder(in)
+                .withType(eu.dnetlib.dhp.oa.graph.hostebymap.model.UnibiGoldModel.class)
+                .build()
+                .parse()
+                .forEach(line ->
+
+                        {
+                            try {
+                                System.out.println(mapper.writeValueAsString(line));
+                            } catch (JsonProcessingException e) {
+                                e.printStackTrace();
+                            }
+                        }
+
+
+                );
+    }
+
+    @Test
+    public void testCSVUrlDOAJ() throws IOException {
+
+        URLConnection connection = new URL("https://doaj.org/csv").openConnection();
+        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
+        connection.connect();
+
+        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
+        //BufferedReader in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
+        PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ_1.csv")));
+        String line = null;
+        while((line = in.readLine())!= null){
+            writer.println(line.replace("\\\"", "\""));
+        }
+        writer.close();
+        in.close();
+        in = new BufferedReader(new FileReader("/tmp/DOAJ_1.csv"));
+        ObjectMapper mapper = new ObjectMapper();
+
+
+
+        new CsvToBeanBuilder(in)
+                .withType(eu.dnetlib.dhp.oa.graph.hostebymap.model.DOAJModel.class)
+                .withMultilineLimit(1)
+                .build()
+                .parse()
+                .forEach(lline ->
+
+                        {
+                            try {
+                                System.out.println(mapper.writeValueAsString(lline));
+                            } catch (JsonProcessingException e) {
+                                e.printStackTrace();
+                            }
+                        }
+
+
+                );
+    }
+}

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt

@@ -1009,7 +1009,7 @@ datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13
 datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN
-datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN
+datacite:id_typologies @=@ datacite:id_typologies @=@ issn @=@ issn
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC
 datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/hostedbymap/doaj_transformed.json

@@ -0,0 +1,25 @@
+{"journalTitle":"Lëd i Sneg","issn":"2076-6734","eissn":"2412-3765","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Компьютерные исследования и моделирование","issn":"2076-7633","eissn":"2077-6853","reviewProcess":"Blind peer review"}
+{"journalTitle":" Историко-биологические исследования","issn":"2076-8176","eissn":"2500-1221","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Інформаційні технології і засоби навчання","issn":"2076-8184","eissn":"","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Revue Internationale de Pédagogie de l’Enseignement Supérieur","issn":"","eissn":"2076-8427","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Проблемы развития территории","issn":"2076-8915","eissn":"2409-9007","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Rambam Maimonides Medical Journal","issn":"","eissn":"2076-9172","reviewProcess":"Peer review"}
+{"journalTitle":"Membranes","issn":"2077-0375","eissn":"","reviewProcess":"Blind peer review"}
+{"journalTitle":"Journal of Clinical Medicine","issn":"","eissn":"2077-0383","reviewProcess":"Blind peer review"}
+{"journalTitle":"Agriculture","issn":"","eissn":"2077-0472","reviewProcess":"Blind peer review"}
+{"journalTitle":"Standartnye Obrazcy","issn":"2077-1177","eissn":"","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Металл и литье Украины","issn":"2077-1304","eissn":"2706-5529","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Journal of Marine Science and Engineering","issn":"","eissn":"2077-1312","reviewProcess":"Blind peer review"}
+{"journalTitle":"Religions","issn":"","eissn":"2077-1444","reviewProcess":"Double blind peer review"}
+{"journalTitle":"GW-Unterricht","issn":"2077-1517","eissn":"2414-4169","reviewProcess":"Double blind peer review"}
+{"journalTitle":"UCV-Scientia","issn":"2077-172X","eissn":"","reviewProcess":"Peer review"}
+{"journalTitle":"Sovremennye Issledovaniâ Socialʹnyh Problem","issn":"2077-1770","eissn":"2218-7405","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Granì","issn":"2077-1800","eissn":"2413-8738","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Journal of Economics Finance and Administrative Science","issn":"2077-1886","eissn":"2218-0648","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Science Education International","issn":"","eissn":"2077-2327","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Edumecentro","issn":"","eissn":"2077-2874","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Monteverdia","issn":"","eissn":"2077-2890","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Transformación","issn":"","eissn":"2077-2955","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Journal of Space Technology","issn":"2077-3099","eissn":"2411-5029","reviewProcess":"Double blind peer review"}
+{"journalTitle":"Revue de Primatologie","issn":"","eissn":"2077-3757","reviewProcess":"Peer review"}

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/hostedbymap/unibiGold.csv

@@ -0,0 +1,37 @@
+"ISSN","ISSN_L","ISSN_IN_DOAJ","ISSN_IN_ROAD","ISSN_IN_PMC","ISSN_IN_OAPC","ISSN_IN_WOS","ISSN_IN_SCOPUS","JOURNAL_IN_DOAJ","JOURNAL_IN_ROAD","JOURNAL_IN_PMC","JOURNAL_IN_OAPC","JOURNAL_IN_WOS","JOURNAL_IN_SCOPUS","TITLE","TITLE_SOURCE"
+"0001-625X","0001-625X",1,1,0,0,0,1,1,1,0,0,0,1,"Acta Mycologica","DOAJ"
+"0002-0397","0002-0397",1,1,0,0,1,1,1,1,0,0,1,1,"Africa Spectrum","DOAJ"
+"0003-2565","0003-2565",1,0,0,0,0,0,1,0,0,0,0,0,"Anali Pravnog Fakulteta u Beogradu","DOAJ"
+"0003-424X","0003-424X",0,1,0,0,1,0,0,1,0,0,1,0,"Annales de zootechnie.","ROAD"
+"0003-4827","0003-4827",0,1,0,0,0,1,0,1,0,0,0,1,"Annals of Iowa.","ROAD"
+"0004-0592","0004-0592",1,1,0,0,1,1,1,1,0,0,1,1,"Archivos de Zootecnia","DOAJ"
+"0004-282X","0004-282X",1,1,0,0,1,1,1,1,0,0,1,1,"Arquivos de Neuro-Psiquiatria","DOAJ"
+"0006-3096","0006-3096",0,1,0,0,0,0,0,1,0,0,0,0,"Biologia.","ROAD"
+"0006-8705","0006-8705",1,1,0,0,1,1,1,1,0,0,1,1,"Bragantia","DOAJ"
+"0007-5124","0007-5124",0,1,0,0,1,0,0,1,1,0,1,1,"Experimental animals.","ROAD"
+"0007-9502","0007-9502",0,1,0,0,0,0,0,1,0,0,0,0,"Caesaraugusta.","ROAD"
+"0008-7386","0008-7386",1,1,0,0,0,1,1,1,0,0,0,1,"Časopis pro Moderní Filologii","DOAJ"
+"0008-7629","0008-7629",1,0,0,0,0,0,1,0,0,0,0,0,"Catalogue and Index","DOAJ"
+"0015-573X","0015-573X",0,1,0,0,0,0,0,1,0,0,0,0,"Folia quaternaria.","ROAD"
+"0016-6987","0016-6987",1,0,0,0,1,1,1,0,0,0,1,1,"Genus","DOAJ"
+"0016-7789","0016-7789",1,1,0,0,0,1,1,1,0,0,0,1,"Geologija ","DOAJ"
+"0021-5007","0021-5007",0,1,0,0,0,1,0,1,0,0,0,1,"Nihon Seitai Gakkaishi.","ROAD"
+"0023-4001","0023-4001",0,1,0,0,1,1,0,1,0,0,1,1,"Korean Journal of Parasitology","ROAD"
+"0023-5415","0023-5415",1,1,0,0,0,0,1,1,0,0,0,0,"Kunst og Kultur","DOAJ"
+"0026-1165","0026-1165",1,0,0,0,1,1,1,0,0,0,1,1,"Journal of the Meteorological Society of Japan","DOAJ"
+"0029-0181","0029-0181",0,1,0,0,0,0,0,1,0,0,0,0,"Nihon butsuri gakkaishi.","ROAD"
+"0034-7000","0034-7000",1,1,0,0,0,1,1,1,0,0,0,1,"Revista Argentina de Cardiología","DOAJ"
+"0034-7523","0034-7523",0,1,0,0,0,1,0,1,0,0,0,1,"Revista cubana de medicina.","ROAD"
+"0034-8244","0034-8244",1,0,0,0,1,1,1,0,0,0,1,1,"Revista de Filosofia","DOAJ"
+"0034-8678","0034-8678",1,0,0,0,0,0,1,0,0,0,0,0,"Revista de Pedagogie","DOAJ"
+"0036-8709","0036-8709",1,1,1,0,1,1,1,1,1,0,1,1,"Scientia Pharmaceutica","DOAJ"
+"0044-4855","0044-4855",0,1,0,0,0,0,0,1,0,0,0,0,"Život i škola.","ROAD"
+"0048-7449","0048-7449",1,1,0,0,1,1,1,1,0,0,1,1,"Reumatismo","DOAJ"
+"0048-766X","0048-766X",0,1,0,0,0,1,0,1,0,0,0,1,"Revista chilena de obstetricia y ginecología.","ROAD"
+"0065-1400","0065-1400",0,1,0,0,1,1,0,1,0,0,1,1,"Acta Neurobiologiae Experimentalis.","ROAD"
+"0066-6742","0066-6742",1,0,0,0,1,1,1,0,0,0,1,1,"Archivo Español de Arqueología","DOAJ"
+"0073-2435","0073-2435",1,1,0,0,1,1,1,1,0,0,1,1,"Historia (Santiago)","DOAJ"
+"0073-4918","0073-4918",0,1,0,0,0,0,0,1,0,0,0,0,"Illinois Natural History Survey bulletin.","ROAD"
+"0075-7411","0075-7411",1,0,0,0,0,0,1,0,0,0,0,0,"Anales","DOAJ"
+"0077-2704","0077-2704",0,1,0,0,0,0,0,1,0,0,0,0,"Namn och bygd.","ROAD"
+"0078-5466","0078-5466",0,1,0,0,1,1,0,1,0,0,1,1,"Optica Applicata.","ROAD"

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/hostedbymap/unibi_transformed.json

@@ -0,0 +1,29 @@
+{"issn":"2502-731X","issn_l":"2502-731X","title":"JIMKESMAS (Jurnal Ilmiah Mahasiswa Kesehatan Masyarakat)","title_source":"ROAD"}
+{"issn":"2502-7409","issn_l":"1411-0253","title":"Jurnal ilmu informasi, perpustakaan, dan kearsipan","title_source":"ROAD"}
+{"issn":"2502-7433","issn_l":"2502-7433","title":"At-Tadbir : jurnal ilmiah manajemen","title_source":"ROAD"}
+{"issn":"2502-745X","issn_l":"2502-745X","title":"Jurnal Kesehatan Panrita Husada.","title_source":"ROAD"}
+{"issn":"2502-7549","issn_l":"2502-7549","title":"ELang journal (An English Education journal)","title_source":"ROAD"}
+{"issn":"2423-3633","issn_l":"2423-3625","title":"̒Ulūm-i darmāngāhī-i dāmpizishkī-i Īrān.","title_source":"ROAD"}
+{"issn":"2423-5563","issn_l":"2423-3773","title":"Pizhūhishnāmah-i  ̒ilm/sanjī.","title_source":"ROAD"}
+{"issn":"1735-434X","issn_l":"1735-434X","title":"Iranian journal of animal biosystematics.","title_source":"ROAD"}
+{"issn":"2423-4435","issn_l":"2008-6113","title":"Majallah-i jangal-i Īrān.","title_source":"ROAD"}
+{"issn":"2423-4575","issn_l":"2423-4575","title":"Ābziyān-i zinatī.","title_source":"ROAD"}
+{"issn":"2423-4974","issn_l":"2423-4974","title":"Pizhūhishnāmah-i ravābiṭ-i  biyn/al- milal.","title_source":"ROAD"}
+{"issn":"2380-0607","issn_l":"2380-0607","title":"AIHM journal club.","title_source":"ROAD"}
+{"issn":"1085-4568","issn_l":"1085-4568","title":"Frontiers.","title_source":"ROAD"}
+{"issn":"2380-8845","issn_l":"2380-8845","title":"˜The œjournal of contemporary archival studies.","title_source":"ROAD"}
+{"issn":"2381-1803","issn_l":"2381-1803","title":"International journal of complementary & alternative medicine.","title_source":"ROAD"}
+{"issn":"2381-2478","issn_l":"2381-2478","title":"Palapala.","title_source":"ROAD"}
+{"issn":"2382-5170","issn_l":"2382-5170","title":"Asia pacific journal of environment ecology and sustainable development.","title_source":"ROAD"}
+{"issn":"2382-9737","issn_l":"2382-9737","title":"Majallah-i salāmat va bihdāsht","title_source":"ROAD"}
+{"issn":"2382-977X","issn_l":"2382-977X","title":"UCT journal of research in science ,engineering and technology","title_source":"ROAD"}
+{"issn":"2382-9974","issn_l":"2382-9974","title":"Bih/nizhādī-i giyāhān-i zirā̒ī va bāghī.","title_source":"ROAD"}
+{"issn":"2227-4782","issn_l":"2227-4782","title":"Problemi endokrinnoï patologìï.","title_source":"ROAD"}
+{"issn":"2685-0079","issn_l":"2597-4971","title":"Jurnal Kebijakan Pembangunan Daerah : Jurnal Penelitian dan Pengembangan Kebijakan Pembangunan Daerah.","title_source":"ROAD"}
+{"issn":"2574-0075","issn_l":"2574-0075","title":"Hypermedia magazine.","title_source":"ROAD"}
+{"issn":"2574-0296","issn_l":"2574-0296","title":"˜The œmuseum review.","title_source":"ROAD"}
+{"issn":"2574-0334","issn_l":"2574-0334","title":"Bioactive compounds in health and disease.","title_source":"ROAD"}
+{"issn":"2574-108X","issn_l":"2574-108X","title":"Journal of computer science integration.","title_source":"ROAD"}
+{"issn":"2574-254X","issn_l":"2574-254X","title":"Child and adolescent obesity.","title_source":"ROAD"}
+{"issn":"2574-3325","issn_l":"2574-3325","title":"Journal of research on the college president.","title_source":"ROAD"}
+{"issn":"2239-6101","issn_l":"2239-5938","title":"European journal of sustainable development.","title_source":"ROAD"}

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_claim_dedup.xml

@@ -16,7 +16,7 @@
         <dc:contributor>Doğuş Üniversitesi, Fen Edebiyat Fakültesi, Fizik Bölümü</dc:contributor>
         <dc:contributor>TR3959</dc:contributor>
         <dc:source>urn:issn:1748-0221</dc:source>
-        <dc:source>VOLUME=7;ISSUE=1;ISSN=1748-0221;TITLE=Journal of Instrumentation</dc:source>
+        <dc:source>VOLUME=7;ISSUE=1;issn=1748-0221;TITLE=Journal of Instrumentation</dc:source>
         <dc:source>ATLAS Collaboration Mitsou, Vasiliki Fiorini, Luca Ros Martínez, Eduardo Castillo
             Giménez, María Victoria Fuster Verdú, Juan A. García García, Carmen Cabrera Urbán,
             Susana Martí García, Salvador Salt Cairols, José Lacasta Llácer, Carlos Valls Ferrer,
@@ -30,7 +30,7 @@
             interactions. Journal of Instrumentation, 7(1). doi: 10.1088/1748-0221/7/01/P01013. UC
             Santa Cruz: Retrieved from: http://www.escholarship.org/uc/item/05j2j2br</dc:source>
         <dc:source>Journal of Instrumentation, 7</dc:source>
-        <dc:source>VOLUME=7;ISSN=1748-0221;TITLE=Journal of Instrumentation</dc:source>
+        <dc:source>VOLUME=7;issn=1748-0221;TITLE=Journal of Instrumentation</dc:source>
         <dc:source>1748-0221</dc:source>
         <dc:source>Journal of Instrumentation 7, P01013 (2012).
             doi:10.1088/1748-0221/7/01/P01013</dc:source>

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/sx/graph/bio/pubmed/pubmed.xml

@@ -16,7 +16,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2944</ISSN>
+                    <issn IssnType="Print">0006-2944</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>13</Volume>
                         <Issue>2</Issue>
@@ -182,7 +182,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1090-2104</ISSN>
+                    <issn IssnType="Electronic">1090-2104</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>66</Volume>
                         <Issue>4</Issue>
@@ -314,7 +314,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-291X</ISSN>
+                    <issn IssnType="Print">0006-291X</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>66</Volume>
                         <Issue>4</Issue>
@@ -460,7 +460,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1090-2104</ISSN>
+                    <issn IssnType="Electronic">1090-2104</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>66</Volume>
                         <Issue>4</Issue>
@@ -644,7 +644,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1090-2104</ISSN>
+                    <issn IssnType="Electronic">1090-2104</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>66</Volume>
                         <Issue>4</Issue>
@@ -774,7 +774,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-291X</ISSN>
+                    <issn IssnType="Print">0006-291X</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>66</Volume>
                         <Issue>4</Issue>
@@ -934,7 +934,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1873-2968</ISSN>
+                    <issn IssnType="Electronic">1873-2968</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>24</Volume>
                         <Issue>16</Issue>
@@ -1614,7 +1614,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1873-2968</ISSN>
+                    <issn IssnType="Electronic">1873-2968</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>24</Volume>
                         <Issue>16</Issue>
@@ -2017,7 +2017,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>17</Issue>
@@ -2185,7 +2185,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Electronic">1873-2968</ISSN>
+                    <issn IssnType="Electronic">1873-2968</issn>
                     <JournalIssue CitedMedium="Internet">
                         <Volume>24</Volume>
                         <Issue>17</Issue>
@@ -2337,7 +2337,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -2585,7 +2585,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -2838,7 +2838,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -3020,7 +3020,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -3204,7 +3204,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -3450,7 +3450,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -3683,7 +3683,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>18</Issue>
@@ -3880,7 +3880,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>20</Issue>
@@ -4069,7 +4069,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>20</Issue>
@@ -4428,7 +4428,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0006-2952</ISSN>
+                    <issn IssnType="Print">0006-2952</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>24</Volume>
                         <Issue>20</Issue>
@@ -4590,7 +4590,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0004-4172</ISSN>
+                    <issn IssnType="Print">0004-4172</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>25</Volume>
                         <Issue>9</Issue>
@@ -4741,7 +4741,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0004-4172</ISSN>
+                    <issn IssnType="Print">0004-4172</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>25</Volume>
                         <Issue>9</Issue>
@@ -4999,7 +4999,7 @@
             </DateRevised>
             <Article PubModel="Print">
                 <Journal>
-                    <ISSN IssnType="Print">0004-4172</ISSN>
+                    <issn IssnType="Print">0004-4172</issn>
                     <JournalIssue CitedMedium="Print">
                         <Volume>25</Volume>
                         <Issue>9</Issue>

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/transform/terms.txt

@@ -1009,7 +1009,7 @@ datacite:id_typologies @=@ datacite:id_typologies @=@ EAN13 @=@ EAN13
 datacite:id_typologies @=@ datacite:id_typologies @=@ EISSN @=@ EISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ Handle @=@ Handle
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISBN @=@ ISBN
-datacite:id_typologies @=@ datacite:id_typologies @=@ ISSN @=@ ISSN
+datacite:id_typologies @=@ datacite:id_typologies @=@ issn @=@ issn
 datacite:id_typologies @=@ datacite:id_typologies @=@ ISTC @=@ ISTC
 datacite:id_typologies @=@ datacite:id_typologies @=@ LISSN @=@ LISSN
 datacite:id_typologies @=@ datacite:id_typologies @=@ LSID @=@ LSID