spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

changed in the constant from propagationconstants to modelconstants

This commit is contained in:
Miriam Baglioni 2020-05-14 18:29:24 +02:00
parent d05630d979
commit f25db01664
6 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareDatasourceCountryAssociation.java
View File

@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
@ -100,7 +101,7 @@ public class PrepareDatasourceCountryAssociation {
+ "JOIN ( SELECT source, target " + "JOIN ( SELECT source, target "
+ " FROM relation " + " FROM relation "
+ " WHERE relclass = '" + " WHERE relclass = '"
+ RELATION_DATASOURCE_ORGANIZATION_REL_CLASS + ModelConstants.IS_PROVIDED_BY
+ "' " + "' "
+ " AND datainfo.deletedbyinference = false ) rel " + " AND datainfo.deletedbyinference = false ) rel "
+ "ON d.id = rel.source " + "ON d.id = rel.source "

3
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/PrepareProjectResultsAssociation.java
View File

@ -8,6 +8,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
@ -85,7 +86,7 @@ public class PrepareProjectResultsAssociation {
+ " FROM relation " + " FROM relation "
+ " WHERE datainfo.deletedbyinference = false " + " WHERE datainfo.deletedbyinference = false "
+ " AND relClass = '" + " AND relClass = '"
+ RELATION_RESULT_PROJECT_REL_CLASS + ModelConstants.IS_PRODUCED_BY
+ "'"; + "'";
Dataset<Row> resproj_relation = spark.sql(resproj_relation_query); Dataset<Row> resproj_relation = spark.sql(resproj_relation_query);

13
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/projecttoresult/SparkResultToProjectThroughSemRelJob.java
View File

@ -9,6 +9,7 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.FlatMapFunction;
@ -122,9 +123,9 @@ public class SparkResultToProjectThroughSemRelJob {
getRelation( getRelation(
resId, resId,
projectId, projectId,
RELATION_RESULT_PROJECT_REL_CLASS, ModelConstants.IS_PRODUCED_BY,
RELATION_RESULTPROJECT_REL_TYPE, ModelConstants.RESULT_PROJECT ,
RELATION_RESULTPROJECT_SUBREL_TYPE, ModelConstants.OUTCOME,
PROPAGATION_DATA_INFO_TYPE, PROPAGATION_DATA_INFO_TYPE,
PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_ID, PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_ID,
PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_NAME)); PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_NAME));
@ -133,9 +134,9 @@ public class SparkResultToProjectThroughSemRelJob {
getRelation( getRelation(
projectId, projectId,
resId, resId,
RELATION_PROJECT_RESULT_REL_CLASS, ModelConstants.PRODUCES,
RELATION_RESULTPROJECT_REL_TYPE, ModelConstants.RESULT_PROJECT,
RELATION_RESULTPROJECT_SUBREL_TYPE, ModelConstants.OUTCOME,
PROPAGATION_DATA_INFO_TYPE, PROPAGATION_DATA_INFO_TYPE,
PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_ID, PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_ID,
PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_NAME)); PROPAGATION_RELATION_RESULT_PROJECT_SEM_REL_CLASS_NAME));

5
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java
View File

@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
import java.util.*; import java.util.*;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapFunction;
@ -74,13 +75,13 @@ public class PrepareResultCommunitySet {
+ " FROM relation " + " FROM relation "
+ " WHERE datainfo.deletedbyinference = false " + " WHERE datainfo.deletedbyinference = false "
+ " AND relClass = '" + " AND relClass = '"
+ RELATION_RESULT_ORGANIZATION_REL_CLASS + ModelConstants.HAS_AUTHOR_INSTITUTION
+ "') result_organization " + "') result_organization "
+ "LEFT JOIN (SELECT source, collect_set(target) org_set " + "LEFT JOIN (SELECT source, collect_set(target) org_set "
+ " FROM relation " + " FROM relation "
+ " WHERE datainfo.deletedbyinference = false " + " WHERE datainfo.deletedbyinference = false "
+ " AND relClass = '" + " AND relClass = '"
+ RELATION_REPRESENTATIVERESULT_RESULT_CLASS + ModelConstants.MERGES
+ "' " + "' "
+ " GROUP BY source) organization_organization " + " GROUP BY source) organization_organization "
+ "ON result_organization.target = organization_organization.source "; + "ON result_organization.target = organization_organization.source ";

5
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/PrepareResultInstRepoAssociation.java
View File

@ -4,6 +4,7 @@ package eu.dnetlib.dhp.resulttoorganizationfrominstrepo;
import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.PropagationConstant.*;
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
@ -91,7 +92,7 @@ public class PrepareResultInstRepoAssociation {
+ "JOIN ( SELECT source, target " + "JOIN ( SELECT source, target "
+ "FROM relation " + "FROM relation "
+ "WHERE relclass = '" + "WHERE relclass = '"
+ RELATION_DATASOURCE_ORGANIZATION_REL_CLASS + ModelConstants.IS_PROVIDED_BY
+ "' " + "' "
+ "AND datainfo.deletedbyinference = false ) rel " + "AND datainfo.deletedbyinference = false ) rel "
+ "ON d.id = rel.source "; + "ON d.id = rel.source ";
@ -111,7 +112,7 @@ public class PrepareResultInstRepoAssociation {
+ "from relation " + "from relation "
+ "where datainfo.deletedbyinference = false " + "where datainfo.deletedbyinference = false "
+ "and relClass = '" + "and relClass = '"
+ RELATION_RESULT_ORGANIZATION_REL_CLASS + ModelConstants.HAS_AUTHOR_INSTITUTION
+ "' " + "' "
+ "group by source"; + "group by source";

13
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob.java
View File

@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession;
import java.util.*; import java.util.*;
import eu.dnetlib.dhp.schema.common.ModelConstants;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
@ -150,9 +151,9 @@ public class SparkResultToOrganizationFromIstRepoJob {
getRelation( getRelation(
orgId, orgId,
resultId, resultId,
RELATION_ORGANIZATION_RESULT_REL_CLASS, ModelConstants.IS_AUTHOR_INSTITUTION_OF,
RELATION_RESULTORGANIZATION_REL_TYPE, ModelConstants.RESULT_ORGANIZATION,
RELATION_RESULTORGANIZATION_SUBREL_TYPE, ModelConstants.AFFILIATION,
PROPAGATION_DATA_INFO_TYPE, PROPAGATION_DATA_INFO_TYPE,
PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_ID, PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_ID,
PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_NAME)); PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_NAME));
@ -161,9 +162,9 @@ public class SparkResultToOrganizationFromIstRepoJob {
getRelation( getRelation(
resultId, resultId,
orgId, orgId,
RELATION_RESULT_ORGANIZATION_REL_CLASS, ModelConstants.HAS_AUTHOR_INSTITUTION,
RELATION_RESULTORGANIZATION_REL_TYPE, ModelConstants.RESULT_ORGANIZATION,
RELATION_RESULTORGANIZATION_SUBREL_TYPE, ModelConstants.AFFILIATION,
PROPAGATION_DATA_INFO_TYPE, PROPAGATION_DATA_INFO_TYPE,
PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_ID, PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_ID,
PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_NAME)); PROPAGATION_RELATION_RESULT_ORGANIZATION_INST_REPO_CLASS_NAME));