From 55e24c25473633e35d3c7721f01b2ab646710b1e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 18 Aug 2020 16:42:08 +0200 Subject: [PATCH] relclass for relation and corresponding values have been put to lower case (isSupplementedBy wrote as IsSupplementedBy - orcid propagation) --- .../eu/dnetlib/dhp/PropagationConstant.java | 4 ++-- .../PrepareDatasourceCountryAssociation.java | 23 +------------------ .../PrepareResultOrcidAssociationStep1.java | 6 +++-- .../PrepareProjectResultsAssociation.java | 6 ++--- .../PrepareResultCommunitySet.java | 8 +++---- .../PrepareResultCommunitySetStep1.java | 4 ++-- .../PrepareResultInstRepoAssociation.java | 8 +++---- 7 files changed, 20 insertions(+), 39 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index c8eb017c7..1cc41c395 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -109,9 +109,9 @@ public class PropagationConstant { } public static String getConstraintList(String text, List constraints) { - String ret = " and (" + text + constraints.get(0) + "'"; + String ret = " and (" + text + constraints.get(0).toLowerCase() + "'"; for (int i = 1; i < constraints.size(); i++) { - ret += " OR " + text + constraints.get(i) + "'"; + ret += " OR " + text + constraints.get(i).toLowerCase() + "'"; } ret += ")"; return ret; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java index f28c5aa06..04a659a1c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java @@ -96,27 +96,6 @@ public class PrepareDatasourceCountryAssociation { relation.createOrReplaceTempView("relation"); organization.createOrReplaceTempView("organization"); -// String query = "SELECT source dataSourceId, named_struct('classid', country.classid, 'classname', country.classname) country " -// + "FROM ( SELECT id " -// + " FROM datasource " -// + " WHERE (datainfo.deletedbyinference = false " -// + whitelisted -// + ") " -// + getConstraintList("datasourcetype.classid = '", allowedtypes) -// + ") d " -// + "JOIN ( SELECT source, target " -// + " FROM relation " -// + " WHERE relclass = '" -// + ModelConstants.IS_PROVIDED_BY -// + "' " -// + " AND datainfo.deletedbyinference = false ) rel " -// + "ON d.id = rel.source " -// + "JOIN (SELECT id, country " -// + " FROM organization " -// + " WHERE datainfo.deletedbyinference = false " -// + " AND length(country.classid) > 0) o " -// + "ON o.id = rel.target"; - String query = "SELECT source dataSourceId, " + "named_struct('classid', country.classid, 'classname', country.classname) country " + "FROM datasource d " + @@ -125,7 +104,7 @@ public class PrepareDatasourceCountryAssociation { "JOIN organization o " + "ON o.id = rel.target " + "WHERE rel.datainfo.deletedbyinference = false " + - "and rel.relclass = '" + ModelConstants.IS_PROVIDED_BY + "'" + + "and lower(rel.relclass) = '" + ModelConstants.IS_PROVIDED_BY.toLowerCase() + "'" + "and o.datainfo.deletedbyinference = false " + "and length(o.country.classid) > 0 " + "and (" + allowed + " or " + whitelisted + ")"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java index b15f813ac..869831ba2 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java @@ -102,15 +102,17 @@ public class PrepareResultOrcidAssociationStep1 { + " FROM result " + " LATERAL VIEW EXPLODE (author) a AS MyT " + " LATERAL VIEW EXPLODE (MyT.pid) p AS MyP " - + " WHERE MyP.qualifier.classid = 'ORCID') tmp " + + " WHERE lower(MyP.qualifier.classid) = 'orcid') tmp " + " GROUP BY id) r_t " + " JOIN (" + " SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + getConstraintList(" relclass = '", allowedsemrel) + + getConstraintList(" lower(relclass) = '", allowedsemrel) + " ) rel_rel " + " ON source = id"; + + log.info("executedQuery: {}", query); spark .sql(query) .as(Encoders.bean(ResultOrcidList.class)) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java index 4cd7f88df..27ff727fd 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java @@ -85,8 +85,8 @@ public class PrepareProjectResultsAssociation { String resproj_relation_query = "SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.IS_PRODUCED_BY + + " AND lower(relClass) = '" + + ModelConstants.IS_PRODUCED_BY.toLowerCase() + "'"; Dataset resproj_relation = spark.sql(resproj_relation_query); @@ -98,7 +98,7 @@ public class PrepareProjectResultsAssociation { + " FROM (SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + getConstraintList(" relClass = '", allowedsemrel) + + getConstraintList(" lower(relClass) = '", allowedsemrel) + " ) r1" + " JOIN resproj_relation r2 " + " ON r1.source = r2.source " diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index 750d333e5..bea0a3e54 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -76,14 +76,14 @@ public class PrepareResultCommunitySet { + "FROM (SELECT source, target " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.HAS_AUTHOR_INSTITUTION + + " AND lower(relClass) = '" + + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() + "') result_organization " + "LEFT JOIN (SELECT source, collect_set(target) org_set " + " FROM relation " + " WHERE datainfo.deletedbyinference = false " - + " AND relClass = '" - + ModelConstants.MERGES + + " AND lower(relClass) = '" + + ModelConstants.MERGES.toLowerCase() + "' " + " GROUP BY source) organization_organization " + "ON result_organization.target = organization_organization.source "; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java index 4f5ac2552..0c836a3ba 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromsemrel/PrepareResultCommunitySetStep1.java @@ -144,8 +144,8 @@ public class PrepareResultCommunitySetStep1 { String resultContextQuery = String .format( RESULT_CONTEXT_QUERY_TEMPLATE, - getConstraintList(" co.id = '", communityIdList), - getConstraintList(" relClass = '", allowedsemrel)); + getConstraintList(" lower(co.id) = '", communityIdList), + getConstraintList(" lower(relClass) = '", allowedsemrel)); Dataset result_context = spark.sql(resultContextQuery); result_context.createOrReplaceTempView("result_context"); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java index 84e40fa88..fe5889c53 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java @@ -91,8 +91,8 @@ public class PrepareResultInstRepoAssociation { + "AND datainfo.deletedbyinference = false ) d " + "JOIN ( SELECT source, target " + "FROM relation " - + "WHERE relclass = '" - + ModelConstants.IS_PROVIDED_BY + + "WHERE lower(relclass) = '" + + ModelConstants.IS_PROVIDED_BY.toLowerCase() + "' " + "AND datainfo.deletedbyinference = false ) rel " + "ON d.id = rel.source "; @@ -111,8 +111,8 @@ public class PrepareResultInstRepoAssociation { String query = "Select source resultId, collect_set(target) organizationSet " + "from relation " + "where datainfo.deletedbyinference = false " - + "and relClass = '" - + ModelConstants.HAS_AUTHOR_INSTITUTION + + "and lower(relClass) = '" + + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() + "' " + "group by source";