diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index c8eb017c71..1cc41c395e 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -109,9 +109,9 @@ public class PropagationConstant { } public static String getConstraintList(String text, List constraints) { - String ret = " and (" + text + constraints.get(0) + "'"; + String ret = " and (" + text + constraints.get(0).toLowerCase() + "'"; for (int i = 1; i < constraints.size(); i++) { - ret += " OR " + text + constraints.get(i) + "'"; + ret += " OR " + text + constraints.get(i).toLowerCase() + "'"; } ret += ")"; return ret; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java index f28c5aa06d..04a659a1ca 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java @@ -96,27 +96,6 @@ public class PrepareDatasourceCountryAssociation { relation.createOrReplaceTempView("relation"); organization.createOrReplaceTempView("organization"); -// String query = "SELECT source dataSourceId, named_struct('classid', country.classid, 'classname', country.classname) country " -// + "FROM ( SELECT id " -// + " FROM datasource " -// + " WHERE (datainfo.deletedbyinference = false " -// + whitelisted -// + ") " -// + getConstraintList("datasourcetype.classid = '", allowedtypes) -// + ") d " -// + "JOIN ( SELECT source, target " -// + " FROM relation " -// + " WHERE relclass = '" -// + ModelConstants.IS_PROVIDED_BY -// + "' " -// + " AND datainfo.deletedbyinference = false ) rel " -// + "ON d.id = rel.source " -// + "JOIN (SELECT id, country " -// + " FROM organization " -// + " WHERE datainfo.deletedbyinference = false " -// + " AND length(country.classid) > 0) o " -// + "ON o.id = rel.target"; - String query = "SELECT source dataSourceId, " + "named_struct('classid', country.classid, 'classname', country.classname) country " + "FROM datasource d " + @@ -125,7 +104,7 @@ public class PrepareDatasourceCountryAssociation { "JOIN organization o " + "ON o.id = rel.target " + "WHERE rel.datainfo.deletedbyinference = false " + - "and rel.relclass = '" + ModelConstants.IS_PROVIDED_BY + "'" + + "and lower(rel.relclass) = '" + ModelConstants.IS_PROVIDED_BY.toLowerCase() + "'" + "and o.datainfo.deletedbyinference = false " + "and length(o.country.classid) > 0 " + "and (" + allowed + " or " + whitelisted + ")"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java index b15f813acd..869831ba2b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java @@ -102,15 +102,17 @@ public class PrepareResultOrcidAssociationStep1 { + " FROM result " + " LATERAL VIEW EXPLODE (author) a AS MyT " + " LATERAL VIEW EXPLODE (MyT.pid) p AS MyP " - + " WHERE MyP.qualifier.classid = 'ORCID') tmp " + + " WHERE lower(MyP.qualifier.classid) = 'orcid') tmp " + " GROUP BY id) r_t " + " JOIN (" + " SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + getConstraintList(" relclass = '", allowedsemrel) + + getConstraintList(" lower(relclass) = '", allowedsemrel) + " ) rel_rel " + " ON source = id"; + + log.info("executedQuery: {}", query); spark .sql(query) .as(Encoders.bean(ResultOrcidList.class)) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java index 4cd7f88dff..27ff727fdf 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java @@ -85,8 +85,8 @@ public class PrepareProjectResultsAssociation { String resproj_relation_query = "SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.IS_PRODUCED_BY + + " AND lower(relClass) = '" + + ModelConstants.IS_PRODUCED_BY.toLowerCase() + "'"; Dataset resproj_relation = spark.sql(resproj_relation_query); @@ -98,7 +98,7 @@ public class PrepareProjectResultsAssociation { + " FROM (SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + getConstraintList(" relClass = '", allowedsemrel) + + getConstraintList(" lower(relClass) = '", allowedsemrel) + " ) r1" + " JOIN resproj_relation r2 " + " ON r1.source = r2.source " diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index 750d333e53..bea0a3e54f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -76,14 +76,14 @@ public class PrepareResultCommunitySet { + "FROM (SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.HAS_AUTHOR_INSTITUTION + + " AND lower(relClass) = '" + + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() + "') result_organization " + "LEFT JOIN (SELECT source, collect_set(target) org_set " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.MERGES + + " AND lower(relClass) = '" + + ModelConstants.MERGES.toLowerCase() + "' " + " GROUP BY source) organization_organization " + "ON result_organization.target = organization_organization.source "; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java index 4f5ac25521..0c836a3ba9 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java @@ -144,8 +144,8 @@ public class PrepareResultCommunitySetStep1 { String resultContextQuery = String .format( RESULT_CONTEXT_QUERY_TEMPLATE, - getConstraintList(" co.id = '", communityIdList), - getConstraintList(" relClass = '", allowedsemrel)); + getConstraintList(" lower(co.id) = '", communityIdList), + getConstraintList(" lower(relClass) = '", allowedsemrel)); Dataset result_context = spark.sql(resultContextQuery); result_context.createOrReplaceTempView("result_context"); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java index 84e40fa88b..fe5889c53f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java @@ -91,8 +91,8 @@ public class PrepareResultInstRepoAssociation { + "AND datainfo.deletedbyinference = false ) d " + "JOIN ( SELECT source, target " + "FROM relation " - + "WHERE relclass = '" - + ModelConstants.IS_PROVIDED_BY + + "WHERE lower(relclass) = '" + + ModelConstants.IS_PROVIDED_BY.toLowerCase() + "' " + "AND datainfo.deletedbyinference = false ) rel " + "ON d.id = rel.source "; @@ -111,8 +111,8 @@ public class PrepareResultInstRepoAssociation { String query = "Select source resultId, collect_set(target) organizationSet " + "from relation " + "where datainfo.deletedbyinference = false " - + "and relClass = '" - + ModelConstants.HAS_AUTHOR_INSTITUTION + + "and lower(relClass) = '" + + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() + "' " + "group by source"; diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml index 5ddc5fedf4..5f52c16585 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/orcidtoresultfromsemrel/oozie_app/workflow.xml @@ -266,7 +266,6 @@ --hive_metastore_uris${hive_metastore_uris} --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication --outputPath${outputPath}/publication - --saveGraph${saveGraph} @@ -298,7 +297,6 @@ --hive_metastore_uris${hive_metastore_uris} --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset --outputPath${outputPath}/dataset - --saveGraph${saveGraph} @@ -330,7 +328,6 @@ --hive_metastore_uris${hive_metastore_uris} --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct --outputPath${outputPath}/otherresearchproduct - --saveGraph${saveGraph} @@ -362,7 +359,6 @@ --hive_metastore_uris${hive_metastore_uris} --resultTableNameeu.dnetlib.dhp.schema.oaf.Software --outputPath${outputPath}/software - --saveGraph${saveGraph} diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml index e0563abae8..2fe9a4256b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/oozie_app/workflow.xml @@ -176,7 +176,6 @@ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked --hive_metastore_uris${hive_metastore_uris} - --saveGraph${saveGraph} --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication @@ -206,7 +205,6 @@ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked --hive_metastore_uris${hive_metastore_uris} - --saveGraph${saveGraph} --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset @@ -236,7 +234,6 @@ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked --hive_metastore_uris${hive_metastore_uris} - --saveGraph${saveGraph} --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct @@ -266,7 +263,6 @@ --datasourceOrganizationPath${workingDir}/preparedInfo/datasourceOrganization --alreadyLinkedPath${workingDir}/preparedInfo/alreadyLinked --hive_metastore_uris${hive_metastore_uris} - --saveGraph${saveGraph} --resultTableNameeu.dnetlib.dhp.schema.oaf.Software diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/DumpGraphEntities.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/DumpGraphEntities.java index 573be6de2e..16653be9dd 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/DumpGraphEntities.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/dump/graph/DumpGraphEntities.java @@ -8,6 +8,8 @@ import java.io.StringReader; import java.util.*; import java.util.stream.Collectors; +import eu.dnetlib.dhp.schema.dump.oaf.graph.Funder; +import eu.dnetlib.dhp.schema.dump.oaf.graph.Project; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Encoders; diff --git a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala index 88277b8274..abac41b89a 100644 --- a/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala +++ b/dhp-workflows/dhp-graph-provision-scholexplorer/src/main/java/eu/dnetlib/dhp/export/DLIToOAF.scala @@ -46,7 +46,7 @@ object DLIToOAF { "IsReferencedBy" -> ("isRelatedTo", "relationship"), "References" -> ("isRelatedTo", "relationship"), "IsRelatedTo" -> ("isRelatedTo", "relationship"), - "IsSupplementedBy" -> ("IsSupplementedBy", "supplement"), + "IsSupplementedBy" -> ("isSupplementedBy", "supplement"), "Cites" -> ("cites", "citation"), "Unknown" -> ("isRelatedTo", "relationship"), "IsSourceOf" -> ("isRelatedTo", "relationship"), diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 192a6f59bf..e2b74b9aa5 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -597,6 +597,12 @@ + + + oozie.launcher.mapreduce.user.classpath.first + true + + eu.dnetlib.dhp.oa.provision.SolrAdminApplication --isLookupUrl${isLookupUrl} --format${format} @@ -639,6 +645,12 @@ + + + oozie.launcher.mapreduce.user.classpath.first + true + + eu.dnetlib.dhp.oa.provision.SolrAdminApplication --isLookupUrl${isLookupUrl} --format${format}