forked from D-Net/dnet-hadoop
Merge branch 'master' of https://code-repo.d4science.org/D-Net/dnet-hadoop into orcid-no-doi
This commit is contained in:
commit
2af21150c5
|
|
@ -38,7 +38,11 @@ import java.io.IOException;
|
|||
import java.sql.Array;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
|
|
@ -174,7 +178,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
|
|||
execute(sqlFile, producer, oaf -> true);
|
||||
}
|
||||
|
||||
public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer,
|
||||
public void execute(final String sqlFile,
|
||||
final Function<ResultSet, List<Oaf>> producer,
|
||||
final Predicate<Oaf> predicate)
|
||||
throws Exception {
|
||||
final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
|
||||
|
|
@ -198,8 +203,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
|
|||
ds
|
||||
.setOriginalId(
|
||||
Arrays
|
||||
.asList(
|
||||
(String[]) rs.getArray("identities").getArray())
|
||||
.asList((String[]) rs.getArray("identities").getArray())
|
||||
.stream()
|
||||
.filter(StringUtils::isNotBlank)
|
||||
.collect(Collectors.toList()));
|
||||
|
|
@ -250,11 +254,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
|
|||
ds
|
||||
.setJournal(
|
||||
journal(
|
||||
rs.getString("officialname"),
|
||||
rs.getString("issnPrinted"),
|
||||
rs.getString("issnOnline"),
|
||||
rs.getString("issnLinking"),
|
||||
info)); // Journal
|
||||
rs.getString("officialname"), rs.getString("issnPrinted"), rs.getString("issnOnline"),
|
||||
rs.getString("issnLinking"), info)); // Journal
|
||||
ds.setDataInfo(info);
|
||||
ds.setLastupdatetimestamp(lastUpdateTimestamp);
|
||||
|
||||
|
|
@ -332,7 +333,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
|
|||
listKeyValues(
|
||||
createOpenaireId(10, rs.getString("collectedfromid"), true),
|
||||
rs.getString("collectedfromname")));
|
||||
o.setPid(new ArrayList<>());
|
||||
o.setPid(prepareListOfStructProps(rs.getArray("pid"), info));
|
||||
o.setDateofcollection(asString(rs.getDate("dateofcollection")));
|
||||
o.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
|
||||
o.setExtraInfo(new ArrayList<>()); // Values not present in the DB
|
||||
|
|
|
|||
|
|
@ -24,12 +24,30 @@ SELECT
|
|||
d.officialname AS collectedfromname,
|
||||
o.country || '@@@dnet:countries' AS country,
|
||||
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
|
||||
ARRAY[]::text[] AS pid
|
||||
|
||||
array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype), NULL) AS pid
|
||||
FROM dsm_organizations o
|
||||
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id)
|
||||
LEFT OUTER JOIN dsm_identities i ON (i.pid = p.pid)
|
||||
GROUP BY
|
||||
o.id,
|
||||
o.legalshortname,
|
||||
o.legalname,
|
||||
o.websiteurl,
|
||||
o.logourl,
|
||||
o.ec_legalbody,
|
||||
o.ec_legalperson,
|
||||
o.ec_nonprofit,
|
||||
o.ec_researchorganization,
|
||||
o.ec_highereducation,
|
||||
o.ec_internationalorganizationeurinterests,
|
||||
o.ec_internationalorganization,
|
||||
o.ec_enterprise,
|
||||
o.ec_smevalidated,
|
||||
o.ec_nutscode,
|
||||
o.dateofcollection,
|
||||
o.lastupdate,
|
||||
o.trust,
|
||||
d.id,
|
||||
d.officialname,
|
||||
o.country
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
|
||||
package eu.dnetlib.dhp.oa.provision;
|
||||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.api.java.function.MapGroupsFunction;
|
||||
import org.apache.spark.sql.*;
|
||||
import org.apache.spark.sql.expressions.Aggregator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
import eu.dnetlib.dhp.common.HdfsSupport;
|
||||
import eu.dnetlib.dhp.oa.provision.model.*;
|
||||
import scala.Tuple2;
|
||||
import scala.collection.JavaConverters;
|
||||
import scala.collection.Seq;
|
||||
|
||||
/**
|
||||
* AdjacencyListBuilderJob: given the tuple (S - R - T) we need to group by S.id -> List [ R - T ], mapping the
|
||||
* result as JoinedEntity
|
||||
*/
|
||||
public class AdjacencyListBuilderJob {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(AdjacencyListBuilderJob.class);
|
||||
|
||||
public static final int MAX_LINKS = 100;
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
||||
IOUtils
|
||||
.toString(
|
||||
AdjacencyListBuilderJob.class
|
||||
.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/oa/provision/input_params_build_adjacency_lists.json")));
|
||||
parser.parseArgument(args);
|
||||
|
||||
Boolean isSparkSessionManaged = Optional
|
||||
.ofNullable(parser.get("isSparkSessionManaged"))
|
||||
.map(Boolean::valueOf)
|
||||
.orElse(Boolean.TRUE);
|
||||
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||
|
||||
String inputPath = parser.get("inputPath");
|
||||
log.info("inputPath: {}", inputPath);
|
||||
|
||||
String outputPath = parser.get("outputPath");
|
||||
log.info("outputPath: {}", outputPath);
|
||||
|
||||
SparkConf conf = new SparkConf();
|
||||
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
|
||||
conf.registerKryoClasses(ProvisionModelSupport.getModelClasses());
|
||||
|
||||
runWithSparkSession(
|
||||
conf,
|
||||
isSparkSessionManaged,
|
||||
spark -> {
|
||||
removeOutputDir(spark, outputPath);
|
||||
createAdjacencyListsKryo(spark, inputPath, outputPath);
|
||||
});
|
||||
}
|
||||
|
||||
private static void createAdjacencyListsKryo(
|
||||
SparkSession spark, String inputPath, String outputPath) {
|
||||
|
||||
log.info("Reading joined entities from: {}", inputPath);
|
||||
|
||||
final List<String> paths = HdfsSupport
|
||||
.listFiles(inputPath, spark.sparkContext().hadoopConfiguration());
|
||||
|
||||
log.info("Found paths: {}", String.join(",", paths));
|
||||
|
||||
}
|
||||
|
||||
private static Seq<String> toSeq(List<String> list) {
|
||||
return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
|
||||
}
|
||||
|
||||
private static void removeOutputDir(SparkSession spark, String path) {
|
||||
HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration());
|
||||
}
|
||||
}
|
||||
|
|
@ -92,7 +92,6 @@ public class CreateRelatedEntitiesJob_phase1 {
|
|||
String outputPath) {
|
||||
|
||||
Dataset<Tuple2<String, Relation>> relsByTarget = readPathRelation(spark, inputRelationsPath)
|
||||
.filter("dataInfo.deletedbyinference == false")
|
||||
.map(
|
||||
(MapFunction<Relation, Tuple2<String, Relation>>) r -> new Tuple2<>(r.getTarget(),
|
||||
r),
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public class CreateRelatedEntitiesJob_phase2 {
|
|||
TypedColumn<JoinedEntity, JoinedEntity> aggregator = new AdjacencyListAggregator().toColumn();
|
||||
|
||||
entities
|
||||
.joinWith(relatedEntities, entities.col("_1").equalTo(relatedEntities.col("_1")), "left_outer")
|
||||
.joinWith(relatedEntities, entities.col("_1").equalTo(relatedEntities.col("_1")), "left")
|
||||
.map((MapFunction<Tuple2<Tuple2<String, E>, Tuple2<String, RelatedEntityWrapper>>, JoinedEntity>) value -> {
|
||||
JoinedEntity je = new JoinedEntity(value._1()._2());
|
||||
Optional
|
||||
|
|
@ -114,7 +114,6 @@ public class CreateRelatedEntitiesJob_phase2 {
|
|||
.ifPresent(r -> je.getLinks().add(r));
|
||||
return je;
|
||||
}, Encoders.kryo(JoinedEntity.class))
|
||||
.filter(filterEmptyEntityFn())
|
||||
.groupByKey(
|
||||
(MapFunction<JoinedEntity, String>) value -> value.getEntity().getId(),
|
||||
Encoders.STRING())
|
||||
|
|
@ -122,7 +121,6 @@ public class CreateRelatedEntitiesJob_phase2 {
|
|||
.map(
|
||||
(MapFunction<Tuple2<String, JoinedEntity>, JoinedEntity>) value -> value._2(),
|
||||
Encoders.kryo(JoinedEntity.class))
|
||||
.filter(filterEmptyEntityFn())
|
||||
.write()
|
||||
.mode(SaveMode.Overwrite)
|
||||
.parquet(outputPath);
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ package eu.dnetlib.dhp.oa.provision;
|
|||
|
||||
import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.HashSet;
|
||||
import java.util.Optional;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
|
@ -15,8 +17,10 @@ import org.apache.spark.api.java.function.FilterFunction;
|
|||
import org.apache.spark.api.java.function.FlatMapFunction;
|
||||
import org.apache.spark.api.java.function.Function;
|
||||
import org.apache.spark.api.java.function.MapFunction;
|
||||
import org.apache.spark.rdd.RDD;
|
||||
import org.apache.spark.sql.*;
|
||||
import org.apache.spark.sql.Encoder;
|
||||
import org.apache.spark.sql.Encoders;
|
||||
import org.apache.spark.sql.SaveMode;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import org.apache.spark.sql.expressions.Aggregator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -24,7 +28,6 @@ import org.slf4j.LoggerFactory;
|
|||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
-------------------------------------------
|
||||
--- Extra tables, mostly used by indicators
|
||||
|
||||
create table ${stats_db_name}.result_projectcount as
|
||||
select r.id, count(distinct p.id) as count
|
||||
from ${stats_db_name}.result r
|
||||
left outer join ${stats_db_name}.result_projects rp on rp.id=r.id
|
||||
left outer join ${stats_db_name}.project p on p.id=rp.project
|
||||
group by r.id;
|
||||
|
||||
create table ${stats_db_name}.result_fundercount as
|
||||
select r.id, count(distinct p.funder) as count
|
||||
from ${stats_db_name}.result r
|
||||
left outer join ${stats_db_name}.result_projects rp on rp.id=r.id
|
||||
left outer join ${stats_db_name}.project p on p.id=rp.project
|
||||
group by r.id;
|
||||
|
||||
create table ${stats_db_name}.project_resultcount as
|
||||
with rcount as (
|
||||
select p.id as pid, count(distinct r.id) as `count`, r.type as type
|
||||
from ${stats_db_name}.project p
|
||||
left outer join ${stats_db_name}.result_projects rp on rp.project=p.id
|
||||
left outer join ${stats_db_name}.result r on r.id=rp.id
|
||||
group by r.type, p.id )
|
||||
select rcount.pid, sum(case when rcount.type='publication' then rcount.count else 0 end) as publications,
|
||||
sum(case when rcount.type='dataset' then rcount.count else 0 end) as datasets,
|
||||
sum(case when rcount.type='software' then rcount.count else 0 end) as software,
|
||||
sum(case when rcount.type='other' then rcount.count else 0 end) as other
|
||||
from rcount
|
||||
group by rcount.pid;
|
||||
|
||||
create view ${stats_db_name}.rndexpenditure as select * from stats_ext.rndexpediture
|
||||
|
|
@ -5,8 +5,12 @@
|
|||
------------------------------------------------------
|
||||
|
||||
-- Dropping old views
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.category;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.concept;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.context;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.countrygdp;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.creation_date;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_classifications;
|
||||
|
|
@ -16,6 +20,7 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_languages;
|
|||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource;
|
||||
|
|
@ -23,11 +28,15 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_languages;
|
|||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_organizations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_results;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.funder;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.fundref;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.numbers_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_projects;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications;
|
||||
|
|
@ -37,12 +46,15 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages;
|
|||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_organizations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_resultcount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results_publication;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_classifications;
|
||||
|
|
@ -52,19 +64,28 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_languages;
|
|||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_affiliated_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_deposited_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_fundercount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_gold;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_greenoa;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_organization;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_peerreviewed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projectcount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projects;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.rndexpediture;
|
||||
|
|
@ -78,6 +99,7 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_languages;
|
|||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_topics;
|
||||
|
||||
|
|
@ -86,8 +108,12 @@ DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_topics;
|
|||
CREATE database IF NOT EXISTS ${stats_db_shadow_name};
|
||||
|
||||
-- Creating new views
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.category AS SELECT * FROM ${stats_db_name}.category;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.concept AS SELECT * FROM ${stats_db_name}.concept;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.context AS SELECT * FROM ${stats_db_name}.context;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.country AS SELECT * FROM ${stats_db_name}.country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.countrygdp AS SELECT * FROM ${stats_db_name}.countrygdp;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.creation_date AS SELECT * FROM ${stats_db_name}.creation_date;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset AS SELECT * FROM ${stats_db_name}.dataset;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_citations AS SELECT * FROM ${stats_db_name}.dataset_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_classifications AS SELECT * FROM ${stats_db_name}.dataset_classifications;
|
||||
|
|
@ -97,6 +123,7 @@ CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_languages AS SELECT *
|
|||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_licenses AS SELECT * FROM ${stats_db_name}.dataset_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_oids AS SELECT * FROM ${stats_db_name}.dataset_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_pids AS SELECT * FROM ${stats_db_name}.dataset_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_refereed AS SELECT * FROM ${stats_db_name}.dataset_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_sources AS SELECT * FROM ${stats_db_name}.dataset_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_topics AS SELECT * FROM ${stats_db_name}.dataset_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource AS SELECT * FROM ${stats_db_name}.datasource;
|
||||
|
|
@ -104,11 +131,15 @@ CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_languages AS SELECT
|
|||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_oids AS SELECT * FROM ${stats_db_name}.datasource_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_organizations AS SELECT * FROM ${stats_db_name}.datasource_organizations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_results AS SELECT * FROM ${stats_db_name}.datasource_results;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_sources AS SELECT * FROM ${stats_db_name}.datasource_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.funder AS SELECT * FROM ${stats_db_name}.funder;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.fundref AS SELECT * FROM ${stats_db_name}.fundref;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.numbers_country AS SELECT * FROM ${stats_db_name}.numbers_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization AS SELECT * FROM ${stats_db_name}.organization;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_datasources AS SELECT * FROM ${stats_db_name}.organization_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_pids AS SELECT * FROM ${stats_db_name}.organization_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_projects AS SELECT * FROM ${stats_db_name}.organization_projects;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_sources AS SELECT * FROM ${stats_db_name}.organization_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct AS SELECT * FROM ${stats_db_name}.otherresearchproduct;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations AS SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications AS SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
|
||||
|
|
@ -118,12 +149,15 @@ CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages
|
|||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses AS SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_refereed AS SELECT * FROM ${stats_db_name}.otherresearchproduct_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics AS SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project AS SELECT * FROM ${stats_db_name}.project;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_oids AS SELECT * FROM ${stats_db_name}.project_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_organizations AS SELECT * FROM ${stats_db_name}.project_organizations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results AS SELECT * FROM ${stats_db_name}.project_results;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_resultcount AS SELECT * FROM ${stats_db_name}.project_resultcount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results_publication AS SELECT * FROM ${stats_db_name}.project_results_publication;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication AS SELECT * FROM ${stats_db_name}.publication;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_citations AS SELECT * FROM ${stats_db_name}.publication_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications;
|
||||
|
|
@ -133,19 +167,28 @@ CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_languages AS SELEC
|
|||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_licenses AS SELECT * FROM ${stats_db_name}.publication_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_oids AS SELECT * FROM ${stats_db_name}.publication_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_pids AS SELECT * FROM ${stats_db_name}.publication_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_refereed AS SELECT * FROM ${stats_db_name}.publication_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_sources AS SELECT * FROM ${stats_db_name}.publication_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_topics AS SELECT * FROM ${stats_db_name}.publication_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result AS SELECT * FROM ${stats_db_name}.result;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_affiliated_country AS SELECT * FROM ${stats_db_name}.result_affiliated_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_citations AS SELECT * FROM ${stats_db_name}.result_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_classifications AS SELECT * FROM ${stats_db_name}.result_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_concepts AS SELECT * FROM ${stats_db_name}.result_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_datasources AS SELECT * FROM ${stats_db_name}.result_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_deposited_country AS SELECT * FROM ${stats_db_name}.result_deposited_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_fundercount AS SELECT * FROM ${stats_db_name}.result_fundercount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_gold AS SELECT * FROM ${stats_db_name}.result_gold;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_greenoa AS SELECT * FROM ${stats_db_name}.result_greenoa;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_languages AS SELECT * FROM ${stats_db_name}.result_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_licenses AS SELECT * FROM ${stats_db_name}.result_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_oids AS SELECT * FROM ${stats_db_name}.result_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_organization AS SELECT * FROM ${stats_db_name}.result_organization;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_peerreviewed AS SELECT * FROM ${stats_db_name}.result_peerreviewed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_pids AS SELECT * FROM ${stats_db_name}.result_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projectcount AS SELECT * FROM ${stats_db_name}.result_projectcount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projects AS SELECT * FROM ${stats_db_name}.result_projects;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_refereed AS SELECT * FROM ${stats_db_name}.result_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_sources AS SELECT * FROM ${stats_db_name}.result_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_topics AS SELECT * FROM ${stats_db_name}.result_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.rndexpediture AS SELECT * FROM ${stats_db_name}.rndexpediture;
|
||||
|
|
@ -159,5 +202,6 @@ CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_languages AS SELECT *
|
|||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_licenses AS SELECT * FROM ${stats_db_name}.software_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_oids AS SELECT * FROM ${stats_db_name}.software_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_pids AS SELECT * FROM ${stats_db_name}.software_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_refereed AS SELECT * FROM ${stats_db_name}.software_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_sources AS SELECT * FROM ${stats_db_name}.software_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_topics AS SELECT * FROM ${stats_db_name}.software_topics;
|
||||
|
|
|
|||
|
|
@ -5,77 +5,4 @@
|
|||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
|
||||
COMPUTE STATS country;
|
||||
COMPUTE STATS countrygdp;
|
||||
COMPUTE STATS dataset;
|
||||
COMPUTE STATS dataset_citations;
|
||||
COMPUTE STATS dataset_classifications;
|
||||
COMPUTE STATS dataset_concepts;
|
||||
COMPUTE STATS dataset_datasources;
|
||||
COMPUTE STATS dataset_languages;
|
||||
COMPUTE STATS dataset_oids;
|
||||
COMPUTE STATS dataset_pids;
|
||||
COMPUTE STATS dataset_sources;
|
||||
COMPUTE STATS dataset_topics;
|
||||
COMPUTE STATS datasource;
|
||||
COMPUTE STATS datasource_languages;
|
||||
COMPUTE STATS datasource_oids;
|
||||
COMPUTE STATS datasource_organizations;
|
||||
COMPUTE STATS datasource_results;
|
||||
COMPUTE STATS fundref;
|
||||
COMPUTE STATS numbers_country;
|
||||
COMPUTE STATS organization;
|
||||
COMPUTE STATS organization_datasources;
|
||||
COMPUTE STATS organization_projects;
|
||||
COMPUTE STATS otherresearchproduct;
|
||||
COMPUTE STATS otherresearchproduct_citations;
|
||||
COMPUTE STATS otherresearchproduct_classifications;
|
||||
COMPUTE STATS otherresearchproduct_concepts;
|
||||
COMPUTE STATS otherresearchproduct_datasources;
|
||||
COMPUTE STATS otherresearchproduct_languages;
|
||||
COMPUTE STATS otherresearchproduct_licenses;
|
||||
COMPUTE STATS otherresearchproduct_oids;
|
||||
COMPUTE STATS otherresearchproduct_pids;
|
||||
COMPUTE STATS otherresearchproduct_sources;
|
||||
COMPUTE STATS otherresearchproduct_topics;
|
||||
COMPUTE STATS project;
|
||||
COMPUTE STATS project_oids;
|
||||
COMPUTE STATS project_organizations;
|
||||
COMPUTE STATS project_results;
|
||||
COMPUTE STATS publication;
|
||||
COMPUTE STATS publication_citations;
|
||||
COMPUTE STATS publication_classifications;
|
||||
COMPUTE STATS publication_concepts;
|
||||
COMPUTE STATS publication_datasources;
|
||||
COMPUTE STATS publication_languages;
|
||||
COMPUTE STATS publication_licenses;
|
||||
COMPUTE STATS publication_oids;
|
||||
COMPUTE STATS publication_pids;
|
||||
COMPUTE STATS publication_sources;
|
||||
COMPUTE STATS publication_topics;
|
||||
COMPUTE STATS result;
|
||||
COMPUTE STATS result_citations;
|
||||
COMPUTE STATS result_classifications;
|
||||
COMPUTE STATS result_concepts;
|
||||
COMPUTE STATS result_datasources;
|
||||
COMPUTE STATS result_languages;
|
||||
COMPUTE STATS result_licenses;
|
||||
COMPUTE STATS result_oids;
|
||||
COMPUTE STATS result_organization;
|
||||
COMPUTE STATS result_pids;
|
||||
COMPUTE STATS result_projects;
|
||||
COMPUTE STATS result_sources;
|
||||
COMPUTE STATS result_topics;
|
||||
COMPUTE STATS rndexpediture;
|
||||
COMPUTE STATS roarmap;
|
||||
COMPUTE STATS software;
|
||||
COMPUTE STATS software_citations;
|
||||
COMPUTE STATS software_classifications;
|
||||
COMPUTE STATS software_concepts;
|
||||
COMPUTE STATS software_datasources;
|
||||
COMPUTE STATS software_languages;
|
||||
COMPUTE STATS software_licenses;
|
||||
COMPUTE STATS software_oids;
|
||||
COMPUTE STATS software_pids;
|
||||
COMPUTE STATS software_sources;
|
||||
COMPUTE STATS software_topics;
|
||||
INVALIDATE METADATA ${stats_db_name};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
-- Impala table statistics - Needed to make the tables
|
||||
-- visible for impala
|
||||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
|
||||
INVALIDATE METADATA ${stats_db_name};
|
||||
|
|
@ -17,19 +17,28 @@ case when size(p.description) > 0 then true else false end as abstract,
|
|||
from ${openaire_db_name}.publication p
|
||||
where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
||||
CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context;
|
||||
CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN (SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||
CREATE TABLE ${stats_db_name}.publication_datasources as
|
||||
SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
|
||||
FROM (
|
||||
SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
|
||||
from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance
|
||||
where p.datainfo.deletedbyinference=false ) p
|
||||
LEFT OUTER JOIN (
|
||||
SELECT substr(d.id, 4) id
|
||||
from ${openaire_db_name}.datasource d
|
||||
WHERE d.datainfo.deletedbyinference=false ) d on p.datasource = d.id;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language FROM ${openaire_db_name}.publication p;
|
||||
CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language FROM ${openaire_db_name}.publication p where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||
CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||
CREATE TABLE ${stats_db_name}.publication_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_topics as select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||
CREATE TABLE ${stats_db_name}.publication_topics as select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.subject) subjects AS subject where p.datainfo.deletedbyinference=false;
|
||||
|
||||
-- Publication_citations
|
||||
CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||
CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="" and p.datainfo.deletedbyinference=false;
|
||||
|
|
@ -17,20 +17,20 @@ FROM ${openaire_db_name}.dataset d
|
|||
WHERE d.datainfo.deletedbyinference=FALSE;
|
||||
|
||||
-- Dataset_citations
|
||||
CREATE TABLE ${stats_db_name}.dataset_citations AS SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.dataset d LATERAL VIEW explode(d.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||
CREATE TABLE ${stats_db_name}.dataset_citations AS SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.dataset d LATERAL VIEW explode(d.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="" and d.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||
CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context;
|
||||
CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_datasources AS SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||
FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
|
||||
FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance) instances AS instance where p.datainfo.deletedbyinference=false) p LEFT OUTER JOIN
|
||||
(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.dataset p;
|
||||
CREATE TABLE ${stats_db_name}.dataset_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.dataset p where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||
CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||
CREATE TABLE ${stats_db_name}.dataset_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||
CREATE TABLE ${stats_db_name}.dataset_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.subject) subjects AS subject where p.datainfo.deletedbyinference=false;
|
||||
|
|
|
|||
|
|
@ -17,20 +17,20 @@ from ${openaire_db_name}.software s
|
|||
where s.datainfo.deletedbyinference=false;
|
||||
|
||||
-- Software_citations
|
||||
CREATE TABLE ${stats_db_name}.software_citations AS SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.software s LATERAL VIEW explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||
CREATE TABLE ${stats_db_name}.software_citations AS SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.software s LATERAL VIEW explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="" and s.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||
CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts AS context;
|
||||
CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts AS context where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||
FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
|
||||
FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance) instances AS instance where p.datainfo.deletedbyinference=false) p LEFT OUTER JOIN
|
||||
(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.software p;
|
||||
CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.software p where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||
CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||
CREATE TABLE ${stats_db_name}.software_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||
CREATE TABLE ${stats_db_name}.software_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.subject) subjects AS subject where p.datainfo.deletedbyinference=false;
|
||||
|
|
|
|||
|
|
@ -17,21 +17,20 @@ FROM ${openaire_db_name}.otherresearchproduct o
|
|||
WHERE o.datainfo.deletedbyinference=FALSE;
|
||||
|
||||
-- Otherresearchproduct_citations
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="" and o.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context;
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||
from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
|
||||
from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance where p.datainfo.deletedbyinference=false) p LEFT OUTER JOIN
|
||||
(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.otherresearchproduct p;
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.otherresearchproduct p where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference=false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject where p.datainfo.deletedbyinference=false;
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@
|
|||
</configuration>
|
||||
</global>
|
||||
|
||||
<start to="Step1"/>
|
||||
<start to="Step18"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
|
|
@ -237,6 +237,17 @@
|
|||
<param>stats_db_name=${stats_db_name}</param>
|
||||
<param>openaire_db_name=${openaire_db_name}</param>
|
||||
</hive2>
|
||||
<ok to="Step16_6"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step16_6">
|
||||
<hive2 xmlns="uri:oozie:hive2-action:0.1">
|
||||
<jdbc-url>${hive_jdbc_url}</jdbc-url>
|
||||
<script>scripts/step16_6.sql</script>
|
||||
<param>stats_db_name=${stats_db_name}</param>
|
||||
<param>openaire_db_name=${openaire_db_name}</param>
|
||||
</hive2>
|
||||
<ok to="Step17"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
|
@ -259,12 +270,26 @@
|
|||
<exec>impala-shell.sh</exec>
|
||||
<argument>${stats_db_name}</argument>
|
||||
<argument>step18.sql</argument>
|
||||
<argument>/user/${wf:user()}/oa/graph/stats/oozie_app/scripts/step18.sql</argument>
|
||||
<argument>${wf:appPath()}/scripts/step18.sql</argument>
|
||||
<file>impala-shell.sh</file>
|
||||
</shell>
|
||||
<ok to="Step19"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step19">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>impala-shell.sh</exec>
|
||||
<argument>${stats_db_shadow_name}</argument>
|
||||
<argument>step19.sql</argument>
|
||||
<argument>${wf:appPath()}/scripts/step19.sql</argument>
|
||||
<file>impala-shell.sh</file>
|
||||
</shell>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
|
|
|
|||
Loading…
Reference in New Issue