Merge branch 'master' of https://code-repo.d4science.org/D-Net/dnet-hadoop

.scalafmt.conf

@@ -0,0 +1,21 @@
+style = defaultWithAlign
+
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.tokens = [{code = "->"}, {code = "<-"}, {code = "=>", owner = "Case"}]
+continuationIndent.callSite = 2
+continuationIndent.defnSite = 2
+danglingParentheses = true
+indentOperator = spray
+maxColumn = 120
+newlines.alwaysBeforeTopLevelStatements = true
+project.excludeFilters = [".*\\.sbt"]
+rewrite.rules = [AvoidInfix]
+rewrite.rules = [ExpandImportSelectors]
+rewrite.rules = [RedundantBraces]
+rewrite.rules = [RedundantParens]
+rewrite.rules = [SortImports]
+rewrite.rules = [SortModifiers]
+rewrite.rules = [PreferCurlyFors]
+spaces.inImportCurlyBraces = false
+unindentTopLevelOperators = true

dhp-workflows/dhp-aggregation/.scalafmt.conf

@@ -0,0 +1,21 @@
+style = defaultWithAlign
+
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.tokens = [{code = "->"}, {code = "<-"}, {code = "=>", owner = "Case"}]
+continuationIndent.callSite = 2
+continuationIndent.defnSite = 2
+danglingParentheses = true
+indentOperator = spray
+maxColumn = 120
+newlines.alwaysBeforeTopLevelStatements = true
+project.excludeFilters = [".*\\.sbt"]
+rewrite.rules = [AvoidInfix]
+rewrite.rules = [ExpandImportSelectors]
+rewrite.rules = [RedundantBraces]
+rewrite.rules = [RedundantParens]
+rewrite.rules = [SortImports]
+rewrite.rules = [SortModifiers]
+rewrite.rules = [PreferCurlyFors]
+spaces.inImportCurlyBraces = false
+unindentTopLevelOperators = true

dhp-workflows/dhp-graph-mapper/.scalafmt.conf

@@ -0,0 +1,21 @@
+style = defaultWithAlign
+
+align.openParenCallSite = false
+align.openParenDefnSite = false
+align.tokens = [{code = "->"}, {code = "<-"}, {code = "=>", owner = "Case"}]
+continuationIndent.callSite = 2
+continuationIndent.defnSite = 2
+danglingParentheses = true
+indentOperator = spray
+maxColumn = 120
+newlines.alwaysBeforeTopLevelStatements = true
+project.excludeFilters = [".*\\.sbt"]
+rewrite.rules = [AvoidInfix]
+rewrite.rules = [ExpandImportSelectors]
+rewrite.rules = [RedundantBraces]
+rewrite.rules = [RedundantParens]
+rewrite.rules = [SortImports]
+rewrite.rules = [SortModifiers]
+rewrite.rules = [PreferCurlyFors]
+spaces.inImportCurlyBraces = false
+unindentTopLevelOperators = true

dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.util.List;
 
+import eu.dnetlib.dhp.schema.oaf.Datasource;
 import org.apache.commons.io.IOUtils;
 import org.dom4j.Document;
 import org.dom4j.DocumentException;
@@ -129,4 +130,33 @@ public class XmlRecordFactoryTest {
 		System.out.println(doc.asXML());
 		assertEquals("", doc.valueOf("//rel/validated"));
 	}
+
+	@Test
+	public void testDatasource() throws IOException, DocumentException {
+		final ContextMapper contextMapper = new ContextMapper();
+
+		final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false,
+				XmlConverterJob.schemaLocation);
+
+		final Datasource d = OBJECT_MAPPER
+				.readValue(IOUtils.toString(getClass().getResourceAsStream("datasource.json")), Datasource.class);
+
+		final String xml = xmlRecordFactory.build(new JoinedEntity<>(d));
+
+		assertNotNull(xml);
+
+		final Document doc = new SAXReader().read(new StringReader(xml));
+
+		assertNotNull(doc);
+
+		System.out.println(doc.asXML());
+
+		// TODO add assertions based of values extracted from the XML record
+
+		assertEquals("National", doc.valueOf("//jurisdiction/@classname"));
+		assertEquals("true", doc.valueOf("//thematic"));
+		assertEquals("Journal article", doc.valueOf("//contentpolicy/@classname"));
+		assertEquals("Journal archive", doc.valueOf("//datasourcetypeui/@classname"));
+
+	}
 }

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql

@@ -14,7 +14,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_sources as 
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -25,7 +25,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_sources as 
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -36,7 +36,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_sources as 
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -47,7 +47,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_sources AS
 SELECT * FROM ${stats_db_name}.publication_sources
@@ -76,8 +76,8 @@ join ${openaire_db_name}.result r1 on rel.source=r1.id
 join ${openaire_db_name}.result r2 on r2.id=rel.target
 where reltype='resultResult'
     and r1.resulttype.classname!=r2.resulttype.classname
-    and r1.datainfo.deletedbyinference=false
-    and r2.datainfo.deletedbyinference=false
+    and r1.datainfo.deletedbyinference=false and r1.datainfo.invisible = FALSE
+    and r2.datainfo.deletedbyinference=false and r2.datainfo.invisible = FALSE
     and r1.resulttype.classname != 'other'
     and r2.resulttype.classname != 'other'
-    and rel.datainfo.deletedbyinference=false;
+    and rel.datainfo.deletedbyinference=false and rel.datainfo.invisible = FALSE;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql

@@ -8,22 +8,22 @@
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_licenses AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_licenses AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_licenses AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_licenses AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
 SELECT * FROM ${stats_db_name}.publication_licenses
@@ -46,7 +46,7 @@ FROM (
     LEFT OUTER JOIN (
         SELECT substr(d.id, 4) id 
         from ${openaire_db_name}.datasource d 
-        WHERE d.datainfo.deletedbyinference=false) d on o.datasource = d.id;
+        WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on o.datasource = d.id;
 
 -- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS;
 -- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql

@@ -9,22 +9,22 @@
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_refereed as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.publication r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_refereed as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.dataset r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_refereed as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.software r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_refereed as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
 select * from ${stats_db_name}.publication_refereed

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql

@@ -38,13 +38,13 @@ SELECT substr(p.id, 4)                                            as id,
        case when size(p.description) > 0 then true else false end as abstract,
        'publication'                                              as type
 from ${openaire_db_name}.publication p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_classifications AS
 SELECT substr(p.id, 4) as id, instancetype.classname as type
 from ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.instance.instancetype) instances as instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_concepts AS
 SELECT substr(p.id, 4) as id, case
@@ -53,45 +53,45 @@ SELECT substr(p.id, 4) as id, case
     when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 from ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.context) contexts as context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_datasources as
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
 FROM (
          SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
          from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     from ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.publication_languages AS
 select substr(p.id, 4) as id, p.language.classname as language
 FROM ${openaire_db_name}.publication p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_oids AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_pids AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_topics as
 select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_citations AS
 SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.publication p
          lateral view explode(p.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and p.datainfo.deletedbyinference = false;
+  and p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql

@@ -81,7 +81,11 @@ compute stats TARGET.result_sources;
 create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_topics;
 
-create table TARGET.result_result stored as parquet as select * from SOURCE.result_result orig where exists (select 1 from TARGET.result r where r.id=orig.source or r.id=orig.target);
+create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result);
+create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result);
+create table TARGET.result_result as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou;
+drop view TARGET.foo1;
+drop view TARGET.foo2;
 compute stats TARGET.result_result;
 
 -- datasources
@@ -126,7 +130,7 @@ compute stats TARGET.indi_result_has_cc_licence;
 create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.indi_result_has_cc_licence_url;
 
-create view TARGET.indi_funder_country_collab stored as select * from SOURCE.indi_funder_country_collab;
+create view TARGET.indi_funder_country_collab stored as parquet as select * from SOURCE.indi_funder_country_collab;
 
 create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.indi_result_with_orcid;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql

@@ -38,20 +38,20 @@ SELECT substr(d.id, 4)                                            AS id,
        CASE WHEN SIZE(d.description) > 0 THEN TRUE ELSE FALSE end AS abstract,
        'dataset'                                                  AS type
 FROM ${openaire_db_name}.dataset d
-WHERE d.datainfo.deletedbyinference = FALSE;
+WHERE d.datainfo.deletedbyinference = FALSE and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_citations AS
 SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.dataset d
          LATERAL VIEW explode(d.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and d.datainfo.deletedbyinference = false;
+  and d.datainfo.deletedbyinference = false and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_classifications AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_concepts AS
 SELECT substr(p.id, 4) as id, case
@@ -60,7 +60,7 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 from ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.context) contexts as context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_datasources AS
 SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource
@@ -68,31 +68,31 @@ FROM (
          SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) AS datasource
          FROM ${openaire_db_name}.dataset p
                   LATERAL VIEW explode(p.instance) instances AS instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     FROM ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d ON p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.dataset_languages AS
 SELECT substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.dataset p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_oids AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_pids AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_topics AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql

@@ -38,20 +38,20 @@ SELECT substr(s.id, 4)                                            as id,
        CASE WHEN SIZE(s.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
        'software'                                                 as type
 from ${openaire_db_name}.software s
-where s.datainfo.deletedbyinference = false;
+where s.datainfo.deletedbyinference = false and s.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_citations AS
 SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.software s
          LATERAL VIEW explode(s.extrainfo) citations as citation
 where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and s.datainfo.deletedbyinference = false;
+  and s.datainfo.deletedbyinference = false and s.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_classifications AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_concepts AS
 SELECT substr(p.id, 4) as id, case
@@ -60,7 +60,7 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.context) contexts AS context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_datasources AS
 SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource
@@ -68,31 +68,31 @@ FROM (
          SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
          FROM ${openaire_db_name}.software p
                   LATERAL VIEW explode(p.instance) instances AS instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     FROM ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d ON p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.software_languages AS
 select substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.software p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_oids AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_pids AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_topics AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql

@@ -37,19 +37,19 @@ SELECT substr(o.id, 4)                                            AS id,
        CASE WHEN SIZE(o.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
        'other'                                                    AS type
 FROM ${openaire_db_name}.otherresearchproduct o
-WHERE o.datainfo.deletedbyinference = FALSE;
+WHERE o.datainfo.deletedbyinference = FALSE and o.datainfo.invisible=false;
 
 -- Otherresearchproduct_citations
 CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS
 SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and o.datainfo.deletedbyinference = false;
+  and o.datainfo.deletedbyinference = false and o.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS
 SELECT substr(p.id, 4) as id, case
@@ -57,33 +57,33 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+::[^::]+$' then concat(contexts.context.id, '::other')
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources AS
 SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource
 FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
       from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance
-      where p.datainfo.deletedbyinference = false) p
+      where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN(SELECT substr(d.id, 4) id
                          from ${openaire_db_name}.datasource d
-                         WHERE d.datainfo.deletedbyinference = false) d on p.datasource = d.id;
+                         WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS
 SELECT substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.otherresearchproduct p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_pids AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql

@@ -5,24 +5,26 @@
 ------------------------------------------------------
 CREATE TABLE ${stats_db_name}.project_oids AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
-FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids;
+FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids
+where p.datainfo.deletedbyinference=false  and p.datainfo.invisible=false;
+
 CREATE TABLE ${stats_db_name}.project_organizations AS
 SELECT substr(r.source, 4) AS id, substr(r.target, 4) AS organization
 from ${openaire_db_name}.relation r
 WHERE r.reltype = 'projectOrganization'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.project_results AS
 SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result, r.datainfo.provenanceaction.classname as provenance
 FROM ${openaire_db_name}.relation r
 WHERE r.reltype = 'resultProject'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 create table ${stats_db_name}.project_classification as
 select substr(p.id, 4) as id, class.h2020programme.code, class.level1, class.level2, class.level3
 from ${openaire_db_name}.project p
     lateral view explode(p.h2020classification) classifs as class
-where p.datainfo.deletedbyinference=false and class.h2020programme is not null;
+where p.datainfo.deletedbyinference=false and p.datainfo.invisible=false and class.h2020programme is not null;
 
 CREATE TABLE ${stats_db_name}.project_tmp
 (
@@ -72,7 +74,7 @@ SELECT substr(p.id, 4)                                                 AS id,
        p.code.value                                                    AS code,
        p.totalcost                                                     AS totalcost
 FROM ${openaire_db_name}.project p
-WHERE p.datainfo.deletedbyinference = false;
+WHERE p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 create table ${stats_db_name}.funder as
 select distinct xpath_string(fund, '//funder/id')        as id,

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql

@@ -127,7 +127,7 @@ CREATE TABLE ${stats_db_name}.result_organization AS
 SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization
 FROM ${openaire_db_name}.relation r
 WHERE r.reltype = 'resultOrganization'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.result_projects AS
 select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend, pr.provenance as provenance

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql

@@ -44,7 +44,7 @@ FROM ${openaire_db_name}.datasource d1
                LATERAL VIEW EXPLODE(originalid) temp AS originalidd
       WHERE originalidd like "piwik:%") AS d2
      ON d1.id = d2.id
-WHERE d1.datainfo.deletedbyinference = FALSE;
+WHERE d1.datainfo.deletedbyinference = FALSE and d1.datainfo.invisible=false;
 
 -- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table.
 -- Creating a temporary dual table that will be removed after the following insert

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql

@@ -9,7 +9,7 @@ SELECT substr(o.id, 4)        as id,
        o.legalshortname.value as legalshortname,
        o.country.classid      as country
 FROM ${openaire_db_name}.organization o
-WHERE o.datainfo.deletedbyinference = FALSE;
+WHERE o.datainfo.deletedbyinference = FALSE and o.datainfo.invisible = FALSE;
 
 CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS
 SELECT organization AS id, id AS datasource

dhp-workflows/pom.xml

@@ -44,7 +44,7 @@
         <pluginRepository>
             <id>iis-releases</id>
             <name>iis releases plugin repository</name>
-            <url>http://maven.ceon.pl/artifactory/iis-releases</url>
+            <url>https://maven.ceon.pl/artifactory/iis-releases</url>
             <layout>default</layout>
         </pluginRepository>
     </pluginRepositories>