merge branch with fork master
This commit is contained in:
commit
8ad8dac7d4
|
|
@ -1,15 +1,22 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.utils;
|
package eu.dnetlib.dhp.utils;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import java.util.Map;
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
|
import javax.xml.ws.BindingProvider;
|
||||||
|
|
||||||
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
|
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||||
|
|
||||||
public class ISLookupClientFactory {
|
public class ISLookupClientFactory {
|
||||||
|
|
||||||
private static final Log log = LogFactory.getLog(ISLookupClientFactory.class);
|
private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
|
||||||
|
|
||||||
|
private static int requestTimeout = 60000 * 10;
|
||||||
|
private static int connectTimeout = 60000 * 10;
|
||||||
|
|
||||||
public static ISLookUpService getLookUpService(final String isLookupUrl) {
|
public static ISLookUpService getLookUpService(final String isLookupUrl) {
|
||||||
return getServiceStub(ISLookUpService.class, isLookupUrl);
|
return getServiceStub(ISLookUpService.class, isLookupUrl);
|
||||||
|
|
@ -21,6 +28,25 @@ public class ISLookupClientFactory {
|
||||||
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
|
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
|
||||||
jaxWsProxyFactory.setServiceClass(clazz);
|
jaxWsProxyFactory.setServiceClass(clazz);
|
||||||
jaxWsProxyFactory.setAddress(endpoint);
|
jaxWsProxyFactory.setAddress(endpoint);
|
||||||
return (T) jaxWsProxyFactory.create();
|
|
||||||
|
final T service = (T) jaxWsProxyFactory.create();
|
||||||
|
|
||||||
|
if (service instanceof BindingProvider) {
|
||||||
|
log
|
||||||
|
.info(
|
||||||
|
"setting timeouts for {} to requestTimeout: {}, connectTimeout: {}",
|
||||||
|
BindingProvider.class.getName(), requestTimeout, connectTimeout);
|
||||||
|
|
||||||
|
Map<String, Object> requestContext = ((BindingProvider) service).getRequestContext();
|
||||||
|
|
||||||
|
requestContext.put("com.sun.xml.internal.ws.request.timeout", requestTimeout);
|
||||||
|
requestContext.put("com.sun.xml.internal.ws.connect.timeout", connectTimeout);
|
||||||
|
requestContext.put("com.sun.xml.ws.request.timeout", requestTimeout);
|
||||||
|
requestContext.put("com.sun.xml.ws.connect.timeout", connectTimeout);
|
||||||
|
requestContext.put("javax.xml.ws.client.receiveTimeout", requestTimeout);
|
||||||
|
requestContext.put("javax.xml.ws.client.connectionTimeout", connectTimeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
return service;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -48,12 +48,13 @@ public class IndexOnESJob {
|
||||||
|
|
||||||
final JavaRDD<String> inputRdd = ClusterUtils
|
final JavaRDD<String> inputRdd = ClusterUtils
|
||||||
.readPath(spark, eventsPath, Event.class)
|
.readPath(spark, eventsPath, Event.class)
|
||||||
// .limit(10000) // TODO REMOVE
|
|
||||||
.map(IndexOnESJob::eventAsJsonString, Encoders.STRING())
|
.map(IndexOnESJob::eventAsJsonString, Encoders.STRING())
|
||||||
.javaRDD();
|
.javaRDD();
|
||||||
|
|
||||||
final Map<String, String> esCfg = new HashMap<>();
|
final Map<String, String> esCfg = new HashMap<>();
|
||||||
// esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54");
|
// esCfg.put("es.nodes", "10.19.65.51, 10.19.65.52, 10.19.65.53, 10.19.65.54");
|
||||||
|
|
||||||
|
esCfg.put("es.index.auto.create", "false");
|
||||||
esCfg.put("es.nodes", indexHost);
|
esCfg.put("es.nodes", indexHost);
|
||||||
esCfg.put("es.mapping.id", "eventId"); // THE PRIMARY KEY
|
esCfg.put("es.mapping.id", "eventId"); // THE PRIMARY KEY
|
||||||
esCfg.put("es.batch.write.retry.count", "8");
|
esCfg.put("es.batch.write.retry.count", "8");
|
||||||
|
|
|
||||||
|
|
@ -64,37 +64,11 @@
|
||||||
</configuration>
|
</configuration>
|
||||||
</global>
|
</global>
|
||||||
|
|
||||||
<start to="generate_events"/>
|
<start to="index_es"/>
|
||||||
|
|
||||||
<kill name="Kill">
|
<kill name="Kill">
|
||||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||||
</kill>
|
</kill>
|
||||||
|
|
||||||
<action name="generate_events">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>GenerateEventsJob</name>
|
|
||||||
<class>eu.dnetlib.dhp.broker.oa.GenerateEventsJob</class>
|
|
||||||
<jar>dhp-broker-events-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.shuffle.partitions=3840
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--workingPath</arg><arg>${workingPath}</arg>
|
|
||||||
<arg>--datasourceIdWhitelist</arg><arg>${datasourceIdWhitelist}</arg>
|
|
||||||
<arg>--datasourceTypeWhitelist</arg><arg>${datasourceTypeWhitelist}</arg>
|
|
||||||
<arg>--datasourceIdBlacklist</arg><arg>${datasourceIdBlacklist}</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="index_es"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="index_es">
|
<action name="index_es">
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||||
|
|
@ -117,34 +91,10 @@
|
||||||
<arg>--index</arg><arg>${esIndexName}</arg>
|
<arg>--index</arg><arg>${esIndexName}</arg>
|
||||||
<arg>--esHost</arg><arg>${esIndexHost}</arg>
|
<arg>--esHost</arg><arg>${esIndexHost}</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="stats"/>
|
|
||||||
<error to="Kill"/>
|
|
||||||
</action>
|
|
||||||
|
|
||||||
<action name="stats">
|
|
||||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
|
||||||
<master>yarn</master>
|
|
||||||
<mode>cluster</mode>
|
|
||||||
<name>GenerateStatsJob</name>
|
|
||||||
<class>eu.dnetlib.dhp.broker.oa.GenerateStatsJob</class>
|
|
||||||
<jar>dhp-broker-events-${projectVersion}.jar</jar>
|
|
||||||
<spark-opts>
|
|
||||||
--executor-cores=${sparkExecutorCores}
|
|
||||||
--executor-memory=${sparkExecutorMemory}
|
|
||||||
--driver-memory=${sparkDriverMemory}
|
|
||||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
|
||||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
|
||||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
|
||||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
|
||||||
--conf spark.sql.shuffle.partitions=3840
|
|
||||||
</spark-opts>
|
|
||||||
<arg>--graphPath</arg><arg>${graphInputPath}</arg>
|
|
||||||
<arg>--workingPath</arg><arg>${workingPath}</arg>
|
|
||||||
</spark>
|
|
||||||
<ok to="End"/>
|
<ok to="End"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
</action>
|
</action>
|
||||||
|
|
||||||
<end name="End"/>
|
<end name="End"/>
|
||||||
|
|
||||||
</workflow-app>
|
</workflow-app>
|
||||||
|
|
@ -9,6 +9,7 @@ import java.util.Optional;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.spark.SparkConf;
|
import org.apache.spark.SparkConf;
|
||||||
import org.apache.spark.api.java.function.MapFunction;
|
import org.apache.spark.api.java.function.MapFunction;
|
||||||
|
import org.apache.spark.sql.Dataset;
|
||||||
import org.apache.spark.sql.Encoders;
|
import org.apache.spark.sql.Encoders;
|
||||||
import org.apache.spark.sql.SaveMode;
|
import org.apache.spark.sql.SaveMode;
|
||||||
import org.apache.spark.sql.SparkSession;
|
import org.apache.spark.sql.SparkSession;
|
||||||
|
|
@ -42,6 +43,12 @@ public class GraphHiveTableImporterJob {
|
||||||
.orElse(Boolean.TRUE);
|
.orElse(Boolean.TRUE);
|
||||||
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
log.info("isSparkSessionManaged: {}", isSparkSessionManaged);
|
||||||
|
|
||||||
|
int numPartitions = Optional
|
||||||
|
.ofNullable(parser.get("numPartitions"))
|
||||||
|
.map(Integer::valueOf)
|
||||||
|
.orElse(-1);
|
||||||
|
log.info("numPartitions: {}", numPartitions);
|
||||||
|
|
||||||
String inputPath = parser.get("inputPath");
|
String inputPath = parser.get("inputPath");
|
||||||
log.info("inputPath: {}", inputPath);
|
log.info("inputPath: {}", inputPath);
|
||||||
|
|
||||||
|
|
@ -60,16 +67,21 @@ public class GraphHiveTableImporterJob {
|
||||||
conf.set("hive.metastore.uris", hiveMetastoreUris);
|
conf.set("hive.metastore.uris", hiveMetastoreUris);
|
||||||
|
|
||||||
runWithSparkHiveSession(
|
runWithSparkHiveSession(
|
||||||
conf, isSparkSessionManaged, spark -> loadGraphTable(spark, inputPath, hiveDbName, clazz));
|
conf, isSparkSessionManaged, spark -> loadGraphTable(spark, inputPath, hiveDbName, clazz, numPartitions));
|
||||||
}
|
}
|
||||||
|
|
||||||
// protected for testing
|
// protected for testing
|
||||||
private static <T extends Oaf> void loadGraphTable(SparkSession spark, String inputPath, String hiveDbName,
|
private static <T extends Oaf> void loadGraphTable(SparkSession spark, String inputPath, String hiveDbName,
|
||||||
Class<T> clazz) {
|
Class<T> clazz, int numPartitions) {
|
||||||
|
|
||||||
spark
|
Dataset<String> dataset = spark.read().textFile(inputPath);
|
||||||
.read()
|
|
||||||
.textFile(inputPath)
|
if (numPartitions > 0) {
|
||||||
|
log.info("repartitioning {} to {} partitions", clazz.getSimpleName(), numPartitions);
|
||||||
|
dataset = dataset.repartition(numPartitions);
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset
|
||||||
.map((MapFunction<String, T>) s -> OBJECT_MAPPER.readValue(s, clazz), Encoders.bean(clazz))
|
.map((MapFunction<String, T>) s -> OBJECT_MAPPER.readValue(s, clazz), Encoders.bean(clazz))
|
||||||
.write()
|
.write()
|
||||||
.mode(SaveMode.Overwrite)
|
.mode(SaveMode.Overwrite)
|
||||||
|
|
|
||||||
|
|
@ -93,9 +93,16 @@ public abstract class AbstractMdRecordToOafMapper {
|
||||||
.stream()
|
.stream()
|
||||||
.map(i -> i.getInstancetype().getClassid())
|
.map(i -> i.getInstancetype().getClassid())
|
||||||
.findFirst()
|
.findFirst()
|
||||||
|
.map(s -> UNKNOWN.equalsIgnoreCase(s) ? "0000" : s)
|
||||||
.orElse("0000"); // Unknown
|
.orElse("0000"); // Unknown
|
||||||
Qualifier resultType = vocs.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, instanceType);
|
return Optional
|
||||||
return resultType.getClassid();
|
.ofNullable(vocs.getSynonymAsQualifier(ModelConstants.DNET_RESULT_TYPOLOGIES, instanceType))
|
||||||
|
.map(q -> q.getClassid())
|
||||||
|
.orElse("0000");
|
||||||
|
/*
|
||||||
|
* .orElseThrow( () -> new IllegalArgumentException( String.format("'%s' not mapped in %s", instanceType,
|
||||||
|
* DNET_RESULT_TYPOLOGIES)));
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
return type;
|
return type;
|
||||||
|
|
|
||||||
|
|
@ -282,6 +282,7 @@
|
||||||
<arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
|
<arg>--hiveDbName</arg><arg>${hiveDbName}</arg>
|
||||||
<arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
|
<arg>--className</arg><arg>eu.dnetlib.dhp.schema.oaf.Project</arg>
|
||||||
<arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
|
<arg>--hiveMetastoreUris</arg><arg>${hiveMetastoreUris}</arg>
|
||||||
|
<arg>--numPartitions</arg><arg>100</arg>
|
||||||
</spark>
|
</spark>
|
||||||
<ok to="join_import"/>
|
<ok to="join_import"/>
|
||||||
<error to="Kill"/>
|
<error to="Kill"/>
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,12 @@
|
||||||
"paramDescription": "when true will stop SparkSession after job execution",
|
"paramDescription": "when true will stop SparkSession after job execution",
|
||||||
"paramRequired": false
|
"paramRequired": false
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"paramName": "np",
|
||||||
|
"paramLongName": "numPartitions",
|
||||||
|
"paramDescription": "number of dataset partitions",
|
||||||
|
"paramRequired": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"paramName": "in",
|
"paramName": "in",
|
||||||
"paramLongName": "inputPath",
|
"paramLongName": "inputPath",
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
|
||||||
import static org.mockito.Mockito.lenient;
|
import static org.mockito.Mockito.lenient;
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
@ -21,8 +19,9 @@ import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
import org.mockito.junit.jupiter.MockitoExtension;
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
import eu.dnetlib.dhp.oa.graph.clean.CleaningFunctionTest;
|
import eu.dnetlib.dhp.oa.graph.clean.CleaningFunctionTest;
|
||||||
import eu.dnetlib.dhp.oa.graph.raw.common.OafMapperUtils;
|
|
||||||
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
|
import eu.dnetlib.dhp.oa.graph.raw.common.VocabularyGroup;
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
import eu.dnetlib.dhp.schema.oaf.Author;
|
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||||
|
|
@ -266,6 +265,17 @@ public class MappersTest {
|
||||||
assertTrue(s.getInstance().size() > 0);
|
assertTrue(s.getInstance().size() > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// @Test
|
||||||
|
void testDataset_2() throws IOException {
|
||||||
|
final String xml = IOUtils.toString(getClass().getResourceAsStream("odf_dataset_2.xml"));
|
||||||
|
|
||||||
|
final List<Oaf> list = new OdfToOafMapper(vocs, false).processMdRecord(xml);
|
||||||
|
|
||||||
|
System.out.println("***************");
|
||||||
|
System.out.println(new ObjectMapper().writeValueAsString(list));
|
||||||
|
System.out.println("***************");
|
||||||
|
}
|
||||||
|
|
||||||
private void assertValidId(final String id) {
|
private void assertValidId(final String id) {
|
||||||
assertEquals(49, id.length());
|
assertEquals(49, id.length());
|
||||||
assertEquals('|', id.charAt(2));
|
assertEquals('|', id.charAt(2));
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ class TestEBI {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Test
|
// @Test
|
||||||
def testEBIData() = {
|
def testEBIData() = {
|
||||||
SparkAddLinkUpdates.main("-mt local[*] -w /home/sandro/Downloads".split(" "))
|
SparkAddLinkUpdates.main("-mt local[*] -w /home/sandro/Downloads".split(" "))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<oai:record xmlns:dr="http://www.driver-repository.eu/namespace/dr"
|
||||||
|
xmlns:dri="http://www.driver-repository.eu/namespace/dri"
|
||||||
|
xmlns:oaf="http://namespace.openaire.eu/oaf" xmlns:oai="http://www.openarchives.org/OAI/2.0/">
|
||||||
|
<oai:header>
|
||||||
|
<dri:objIdentifier>opentrials__::0000bf8e63d3d7e6b88421eabafae3f6</dri:objIdentifier>
|
||||||
|
<dri:recordIdentifier>feabb67c-1fd1-423b-aec6-606d04ce53c6</dri:recordIdentifier>
|
||||||
|
<dri:dateOfCollection>2019-03-27T15:15:22.22Z</dri:dateOfCollection>
|
||||||
|
<oaf:datasourceprefix>opentrials__</oaf:datasourceprefix>
|
||||||
|
<dr:dateOfTransformation>2019-04-17T16:04:20.586Z</dr:dateOfTransformation>
|
||||||
|
</oai:header>
|
||||||
|
<oai:metadata>
|
||||||
|
<resource xmlns="http://datacite.org/schema/kernel-3"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd">
|
||||||
|
<identifier identifierType="URL">https://clinicaltrials.gov/ct2/show/NCT02321059</identifier>
|
||||||
|
<alternateIdentifiers>
|
||||||
|
<alternateIdentifier alternateIdentifierType="URL">http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059</alternateIdentifier>
|
||||||
|
<alternateIdentifier alternateIdentifierType="nct">NCT02321059</alternateIdentifier>
|
||||||
|
</alternateIdentifiers>
|
||||||
|
<creators>
|
||||||
|
<creator>
|
||||||
|
<creatorName>Jensen, Kristian K</creatorName>
|
||||||
|
</creator>
|
||||||
|
</creators>
|
||||||
|
<titles>
|
||||||
|
<title>Validation of the Goodstrength System for Assessment of Abdominal Wall Strength in Patients With Incisional Hernia</title>
|
||||||
|
</titles>
|
||||||
|
<publisher>nct</publisher>
|
||||||
|
<geoLocations>
|
||||||
|
<geoLocationPlace>Denmark</geoLocationPlace>
|
||||||
|
</geoLocations>
|
||||||
|
<resourceType resourceTypeGeneral="clinicalTrial">0037</resourceType>
|
||||||
|
<descriptions>
|
||||||
|
<description descriptionType="Abstract">Patients with an incisional hernia in the midline and controls with an intact abdominal wall are examined twice with one week apart, in order to establish the test-retest reliability and internal and external validity of the Goodstrength trunk dynamometer.</description>
|
||||||
|
</descriptions>
|
||||||
|
</resource>
|
||||||
|
<oaf:accessrights>OPEN</oaf:accessrights>
|
||||||
|
<dr:CobjCategory type="dataset">0037</dr:CobjCategory>
|
||||||
|
<oaf:dateAccepted>2014-11-11</oaf:dateAccepted>
|
||||||
|
<oaf:hostedBy id="openaire____::opentrials" name="OpenTrials"/>
|
||||||
|
<oaf:collectedFrom id="openaire____::opentrials" name="OpenTrials"/>
|
||||||
|
<oaf:about>
|
||||||
|
<oaf:datainfo>
|
||||||
|
<oaf:inferred>false</oaf:inferred>
|
||||||
|
<oaf:deletedbyinference>false</oaf:deletedbyinference>
|
||||||
|
<oaf:trust>0.9</oaf:trust>
|
||||||
|
<oaf:inferenceprovenance/>
|
||||||
|
<oaf:provenanceaction
|
||||||
|
classid="sysimport:crosswalk:datasetarchive"
|
||||||
|
classname="sysimport:crosswalk:datasetarchive"
|
||||||
|
schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
|
||||||
|
</oaf:datainfo>
|
||||||
|
</oaf:about>
|
||||||
|
</oai:metadata>
|
||||||
|
<about xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||||
|
xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
|
||||||
|
<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">
|
||||||
|
<originDescription altered="true" harvestDate="2019-03-27T15:15:22.22Z">
|
||||||
|
<baseURL>file:///var/lib/dnet/data/opentrials/opentrials.csv</baseURL>
|
||||||
|
<identifier/>
|
||||||
|
<datestamp/>
|
||||||
|
<metadataNamespace/>
|
||||||
|
</originDescription>
|
||||||
|
</provenance>
|
||||||
|
<oaf:datainfo>
|
||||||
|
<oaf:inferred>false</oaf:inferred>
|
||||||
|
<oaf:deletedbyinference>false</oaf:deletedbyinference>
|
||||||
|
<oaf:trust>0.9</oaf:trust>
|
||||||
|
<oaf:inferenceprovenance/>
|
||||||
|
<oaf:provenanceaction classid="sysimport:crosswalk:datasetarchive"
|
||||||
|
classname="sysimport:crosswalk:datasetarchive"
|
||||||
|
schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
|
||||||
|
</oaf:datainfo>
|
||||||
|
</about>
|
||||||
|
</oai:record>
|
||||||
|
|
@ -16,15 +16,15 @@
|
||||||
<value>spark2</value>
|
<value>spark2</value>
|
||||||
</property>
|
</property>
|
||||||
<property>
|
<property>
|
||||||
<name>hiveMetastoreUris</name>
|
<name>hive_metastore_uris</name>
|
||||||
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
|
<value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
|
||||||
</property>
|
</property>
|
||||||
<property>
|
<property>
|
||||||
<name>hiveJdbcUrl</name>
|
<name>hive_jdbc_url</name>
|
||||||
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
|
<value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
|
||||||
</property>
|
</property>
|
||||||
<property>
|
<property>
|
||||||
<name>oozie.wf.workflow.notification.url</name>
|
<name>oozie.wf.workflow.notification.url</name>
|
||||||
<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
|
<value>{serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status</value>
|
||||||
</property>
|
</property>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||||
|
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||||
|
if ! [ -L $link_folder ]
|
||||||
|
then
|
||||||
|
rm -Rf "$link_folder"
|
||||||
|
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Getting file from " $3
|
||||||
|
hdfs dfs -copyToLocal $3
|
||||||
|
|
||||||
|
echo "Running impala shell make the new database visible"
|
||||||
|
impala-shell -q "INVALIDATE METADATA;"
|
||||||
|
|
||||||
|
echo "Running impala shell to compute new table stats"
|
||||||
|
impala-shell -d $1 -f $2
|
||||||
|
echo "Impala shell finished"
|
||||||
|
rm $2
|
||||||
|
|
@ -1,11 +1,8 @@
|
||||||
-- DROP database if EXISTS ${hive_db_name} cascade;
|
--------------------------------------------------------------
|
||||||
-- CREATE database ${hive_db_name};
|
--------------------------------------------------------------
|
||||||
--
|
-- Stats database creation
|
||||||
-- CREATE TABLE ${hive_db_name}.Persons (
|
--------------------------------------------------------------
|
||||||
-- PersonID int,
|
--------------------------------------------------------------
|
||||||
-- LastName varchar(255));
|
|
||||||
--
|
|
||||||
-- INSERT INTO ${hive_db_name}.Persons VALUES (1, "test_db_spyros_rec_111");
|
|
||||||
|
|
||||||
DROP database IF EXISTS ${stats_db_name} CASCADE;
|
DROP database IF EXISTS ${stats_db_name} CASCADE;
|
||||||
CREATE database ${stats_db_name};
|
CREATE database ${stats_db_name};
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.fundref AS SELECT * FROM ${external_stats_db_name}.fundref;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.country AS SELECT * FROM ${external_stats_db_name}.country;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.countrygdp AS SELECT * FROM ${external_stats_db_name}.countrygdp;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.roarmap AS SELECT * FROM ${external_stats_db_name}.roarmap;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.rndexpediture AS SELECT * FROM ${external_stats_db_name}.rndexpediture;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.context AS SELECT * FROM ${external_stats_db_name}.context;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.category AS SELECT * FROM ${external_stats_db_name}.category;
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.concept AS SELECT * FROM ${external_stats_db_name}.concept;
|
||||||
|
|
||||||
|
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
-- Creation date of the database
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
create table ${stats_db_name}.creation_date as select date_format(current_date(), 'dd-MM-yyyy') as date;
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
----------------------------------------------------------------
|
|
||||||
----------------------------------------------------------------
|
|
||||||
-- Organization table/view and Organization related tables/views
|
|
||||||
----------------------------------------------------------------
|
|
||||||
----------------------------------------------------------------
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.organization;
|
|
||||||
CREATE TABLE ${stats_db_name}.organization AS SELECT substr(o.id, 4) as id, o.legalname.value as name, o.country.classid as country from ${openaire_db_name}.organization o WHERE o.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS SELECT organization AS id, id AS datasource FROM ${stats_db_name}.datasource_organizations;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.organization_projects AS SELECT id AS project, organization as id FROM ${stats_db_name}.project_organizations;
|
|
||||||
|
|
@ -1,10 +1,44 @@
|
||||||
------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)
|
-- Post processing - Updates on main tables
|
||||||
------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
------------------------------------------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.fundref AS SELECT * FROM ${external_stats_db_name}.fundref;
|
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.country AS SELECT * FROM ${external_stats_db_name}.country;
|
--Datasource temporary table updates
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.countrygdp AS SELECT * FROM ${external_stats_db_name}.countrygdp;
|
UPDATE ${stats_db_name}.datasource_tmp SET harvested='true' WHERE datasource_tmp.id IN (SELECT DISTINCT d.id FROM ${stats_db_name}.datasource_tmp d, ${stats_db_name}.result_datasources rd WHERE d.id=rd.datasource);
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.roarmap AS SELECT * FROM ${external_stats_db_name}.roarmap;
|
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.rndexpediture AS SELECT * FROM ${external_stats_db_name}.rndexpediture;
|
-- Project temporary table update and final project table creation with final updates that can not be applied to ORC tables
|
||||||
|
UPDATE ${stats_db_name}.project_tmp SET haspubs='yes' WHERE project_tmp.id IN (SELECT pr.id FROM ${stats_db_name}.project_results pr, ${stats_db_name}.result r WHERE pr.result=r.id AND r.type='publication');
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.project;
|
||||||
|
CREATE TABLE ${stats_db_name}.project stored as parquet as
|
||||||
|
SELECT p.id , p.acronym, p.title, p.funder, p.funding_lvl0, p.funding_lvl1, p.funding_lvl2, p.ec39, p.type, p.startdate, p.enddate, p.start_year, p.end_year, p.duration,
|
||||||
|
CASE WHEN prr1.id IS NULL THEN 'no' ELSE 'yes' END AS haspubs,
|
||||||
|
CASE WHEN prr1.id IS NULL THEN 0 ELSE prr1.np END AS numpubs,
|
||||||
|
CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.daysForlastPub END AS daysforlastpub,
|
||||||
|
CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.dp END AS delayedpubs,
|
||||||
|
p.callidentifier, p.code
|
||||||
|
FROM ${stats_db_name}.project_tmp p
|
||||||
|
LEFT JOIN (SELECT pr.id, count(distinct pr.result) AS np
|
||||||
|
FROM ${stats_db_name}.project_results pr INNER JOIN ${stats_db_name}.result r ON pr.result=r.id
|
||||||
|
WHERE r.type='publication'
|
||||||
|
GROUP BY pr.id) AS prr1 on prr1.id = p.id
|
||||||
|
LEFT JOIN (SELECT pp.id, max(datediff(to_date(r.date), to_date(pp.enddate)) ) AS daysForlastPub , count(distinct r.id) AS dp
|
||||||
|
FROM ${stats_db_name}.project_tmp pp, ${stats_db_name}.project_results pr, ${stats_db_name}.result r
|
||||||
|
WHERE pp.id=pr.id AND pr.result=r.id AND r.type='publication' AND datediff(to_date(r.date), to_date(pp.enddate)) > 0
|
||||||
|
GROUP BY pp.id) AS prr2
|
||||||
|
ON prr2.id = p.id;
|
||||||
|
|
||||||
|
-- Publication temporary table updates
|
||||||
|
UPDATE ${stats_db_name}.publication_tmp SET delayed = 'yes' WHERE publication_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
||||||
|
|
||||||
|
-- Dataset temporary table updates
|
||||||
|
UPDATE ${stats_db_name}.dataset_tmp SET delayed = 'yes' WHERE dataset_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
||||||
|
|
||||||
|
-- Software temporary table updates
|
||||||
|
UPDATE ${stats_db_name}.software_tmp SET delayed = 'yes' WHERE software_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
||||||
|
|
||||||
|
-- Oherresearchproduct temporary table updates
|
||||||
|
UPDATE ${stats_db_name}.otherresearchproduct_tmp SET delayed = 'yes' WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS SELECT result_projects.id AS result, result_projects.project AS project_results, result.date as resultdate, project.enddate as projectenddate, result_projects.daysfromend AS daysfromend FROM ${stats_db_name}.result_projects, ${stats_db_name}.result, ${stats_db_name}.project WHERE result_projects.id=result.id AND result.type='publication' AND project.id=result_projects.project;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
------------------------------------------------------------------------------------------------------
|
||||||
|
-- Creating parquet tables from the updated temporary tables and removing unnecessary temporary tables
|
||||||
|
------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.datasource;
|
||||||
|
CREATE TABLE ${stats_db_name}.datasource stored AS parquet AS SELECT * FROM ${stats_db_name}.datasource_tmp;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.publication;
|
||||||
|
CREATE TABLE ${stats_db_name}.publication stored AS parquet AS SELECT * FROM ${stats_db_name}.publication_tmp;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.dataset;
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset stored AS parquet AS SELECT * FROM ${stats_db_name}.dataset_tmp;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.software;
|
||||||
|
CREATE TABLE ${stats_db_name}.software stored AS parquet AS SELECT * FROM ${stats_db_name}.software_tmp;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct;
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct stored AS parquet AS SELECT * FROM ${stats_db_name}.otherresearchproduct_tmp;
|
||||||
|
|
||||||
|
DROP TABLE ${stats_db_name}.project_tmp;
|
||||||
|
DROP TABLE ${stats_db_name}.datasource_tmp;
|
||||||
|
DROP TABLE ${stats_db_name}.publication_tmp;
|
||||||
|
DROP TABLE ${stats_db_name}.dataset_tmp;
|
||||||
|
DROP TABLE ${stats_db_name}.software_tmp;
|
||||||
|
DROP TABLE ${stats_db_name}.otherresearchproduct_tmp;
|
||||||
|
|
||||||
|
----------------------------------------------
|
||||||
|
-- Re-creating views from final parquet tables
|
||||||
|
---------------------------------------------
|
||||||
|
|
||||||
|
-- Result
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result AS SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.publication UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.software UNION ALL SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.dataset UNION ALL SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.otherresearchproduct;
|
||||||
|
|
||||||
|
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
-- To see with Antonis if the following is needed and where it should be placed
|
||||||
|
-------------------------------------------------------------------------------
|
||||||
|
CREATE TABLE ${stats_db_name}.numbers_country AS SELECT org.country AS country, count(distinct rd.datasource) AS datasources, count(distinct r.id) AS publications FROM ${stats_db_name}.result r, ${stats_db_name}.result_datasources rd, ${stats_db_name}.datasource d, ${stats_db_name}.datasource_organizations dor, ${stats_db_name}.organization org WHERE r.id=rd.id AND rd.datasource=d.id AND d.id=dor.id AND dor.organization=org.id AND r.type='publication' AND r.bestlicence='Open Access' GROUP BY org.country;
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
----------------------------
|
|
||||||
-- Post processing - Updates
|
|
||||||
----------------------------
|
|
||||||
|
|
||||||
--Datasource temporary table updates
|
|
||||||
UPDATE ${stats_db_name}.datasource_tmp set harvested ='true' WHERE datasource_tmp.id IN (SELECT DISTINCT d.id FROM ${stats_db_name}.datasource_tmp d, ${stats_db_name}.result_datasources rd where d.id=rd.datasource);
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Project temporary table update and final project table creation with final updates that can not be applied to ORC tables
|
|
||||||
UPDATE ${stats_db_name}.project_tmp SET haspubs='yes' WHERE project_tmp.id IN (SELECT pr.id FROM ${stats_db_name}.project_results pr, ${stats_db_name}.result r WHERE pr.result=r.id AND r.type='publication');
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.project;
|
|
||||||
|
|
||||||
CREATE TABLE ${stats_db_name}.project stored as parquet as
|
|
||||||
SELECT p.id , p.acronym, p.title, p.funder, p.funding_lvl0, p.funding_lvl1, p.funding_lvl2, p.ec39, p.type, p.startdate, p.enddate, p.start_year, p.end_year, p.duration,
|
|
||||||
CASE WHEN prr1.id IS NULL THEN 'no' ELSE 'yes' END as haspubs,
|
|
||||||
CASE WHEN prr1.id IS NULL THEN 0 ELSE prr1.np END as numpubs,
|
|
||||||
CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.daysForlastPub END as daysforlastpub,
|
|
||||||
CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.dp END as delayedpubs,
|
|
||||||
p.callidentifier, p.code
|
|
||||||
FROM ${stats_db_name}.project_tmp p
|
|
||||||
LEFT JOIN (SELECT pr.id, count(distinct pr.result) AS np
|
|
||||||
FROM ${stats_db_name}.project_results pr INNER JOIN ${stats_db_name}.result r ON pr.result=r.id
|
|
||||||
WHERE r.type='publication'
|
|
||||||
GROUP BY pr.id) AS prr1 on prr1.id = p.id
|
|
||||||
LEFT JOIN (SELECT pp.id, max(datediff(to_date(r.date), to_date(pp.enddate)) ) as daysForlastPub , count(distinct r.id) as dp
|
|
||||||
FROM ${stats_db_name}.project_tmp pp, ${stats_db_name}.project_results pr, ${stats_db_name}.result r
|
|
||||||
WHERE pp.id=pr.id AND pr.result=r.id AND r.type='publication' AND datediff(to_date(r.date), to_date(pp.enddate)) > 0
|
|
||||||
GROUP BY pp.id) AS prr2
|
|
||||||
on prr2.id = p.id;
|
|
||||||
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Publication temporary table updates
|
|
||||||
UPDATE ${stats_db_name}.publication_tmp SET delayed = 'yes' WHERE publication_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Dataset temporary table updates
|
|
||||||
UPDATE ${stats_db_name}.dataset_tmp SET delayed = 'yes' WHERE dataset_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Software temporary table updates
|
|
||||||
UPDATE ${stats_db_name}.software_tmp SET delayed = 'yes' WHERE software_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Oherresearchproduct temporary table updates
|
|
||||||
UPDATE ${stats_db_name}.otherresearchproduct_tmp SET delayed = 'yes' WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS SELECT result_projects.id AS result, result_projects.project AS project_results, result.date as resultdate, project.enddate as projectenddate, result_projects.daysfromend as daysfromend FROM ${stats_db_name}.result_projects, ${stats_db_name}.result, ${stats_db_name}.project WHERE result_projects.id=result.id and result.type='publication' and project.id=result_projects.project;
|
|
||||||
|
|
@ -1,26 +1,59 @@
|
||||||
------------------------------------------------------------------------------------------------------
|
------------------------------------------------------
|
||||||
-- Creating parquet tables from the updated temporary tables and removing unnecessary temporary tables
|
------------------------------------------------------
|
||||||
------------------------------------------------------------------------------------------------------
|
-- Additional relations
|
||||||
|
--
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.datasource;
|
-- Sources related tables/views
|
||||||
CREATE TABLE ${stats_db_name}.datasource stored as parquet as select * from ${stats_db_name}.datasource_tmp;
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.publication;
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_sources as
|
||||||
CREATE TABLE ${stats_db_name}.publication stored as parquet as select * from ${stats_db_name}.publication_tmp;
|
SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
|
||||||
|
FROM (
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.dataset;
|
SELECT substr(p.id, 4) as id, substr(datasource, 4) as datasource
|
||||||
CREATE TABLE ${stats_db_name}.dataset stored as parquet as select * from ${stats_db_name}.dataset_tmp;
|
from ${openaire_db_name}.publication p lateral view explode(p.collectedfrom.key) c as datasource) p
|
||||||
|
LEFT OUTER JOIN
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.software;
|
(
|
||||||
CREATE TABLE ${stats_db_name}.software stored as parquet as select * from ${stats_db_name}.software_tmp;
|
SELECT substr(d.id, 4) id
|
||||||
|
from ${openaire_db_name}.datasource d
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct;
|
WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct stored as parquet as select * from ${stats_db_name}.otherresearchproduct_tmp;
|
|
||||||
|
|
||||||
DROP TABLE ${stats_db_name}.project_tmp;
|
|
||||||
DROP TABLE ${stats_db_name}.datasource_tmp;
|
|
||||||
DROP TABLE ${stats_db_name}.publication_tmp;
|
|
||||||
DROP TABLE ${stats_db_name}.dataset_tmp;
|
|
||||||
DROP TABLE ${stats_db_name}.software_tmp;
|
|
||||||
DROP TABLE ${stats_db_name}.otherresearchproduct_tmp;
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_sources as
|
||||||
|
SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
|
||||||
|
FROM (
|
||||||
|
SELECT substr(p.id, 4) as id, substr(datasource, 4) as datasource
|
||||||
|
from ${openaire_db_name}.dataset p lateral view explode(p.collectedfrom.key) c as datasource) p
|
||||||
|
LEFT OUTER JOIN
|
||||||
|
(
|
||||||
|
SELECT substr(d.id, 4) id
|
||||||
|
from ${openaire_db_name}.datasource d
|
||||||
|
WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_sources as
|
||||||
|
SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
|
||||||
|
FROM (
|
||||||
|
SELECT substr(p.id, 4) as id, substr(datasource, 4) as datasource
|
||||||
|
from ${openaire_db_name}.software p lateral view explode(p.collectedfrom.key) c as datasource) p
|
||||||
|
LEFT OUTER JOIN
|
||||||
|
(
|
||||||
|
SELECT substr(d.id, 4) id
|
||||||
|
from ${openaire_db_name}.datasource d
|
||||||
|
WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_sources as
|
||||||
|
SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
|
||||||
|
FROM (
|
||||||
|
SELECT substr(p.id, 4) as id, substr(datasource, 4) as datasource
|
||||||
|
from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.collectedfrom.key) c as datasource) p
|
||||||
|
LEFT OUTER JOIN
|
||||||
|
(
|
||||||
|
SELECT substr(d.id, 4) id
|
||||||
|
from ${openaire_db_name}.datasource d
|
||||||
|
WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_sources AS
|
||||||
|
SELECT * FROM ${stats_db_name}.publication_sources
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.dataset_sources
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.software_sources
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,49 @@
|
||||||
----------------------------------------------
|
------------------------------------------------------
|
||||||
-- Re-creating views from final parquet tables
|
------------------------------------------------------
|
||||||
---------------------------------------------
|
-- Additional relations
|
||||||
|
--
|
||||||
|
-- Licences related tables/views
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_licenses AS
|
||||||
|
SELECT substr(p.id, 4) as id, licenses.value as type
|
||||||
|
from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.license) instances as licenses
|
||||||
|
where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
-- Result
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_licenses AS
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence as access_mode FROM ${stats_db_name}.publication UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.software UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.dataset UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.otherresearchproduct;
|
SELECT substr(p.id, 4) as id, licenses.value as type
|
||||||
|
from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.license) instances as licenses
|
||||||
|
where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_licenses AS
|
||||||
|
SELECT substr(p.id, 4) as id, licenses.value as type
|
||||||
|
from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.license) instances as licenses
|
||||||
|
where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_licenses AS
|
||||||
|
SELECT substr(p.id, 4) as id, licenses.value as type
|
||||||
|
from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
|
||||||
|
where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
|
||||||
|
SELECT * FROM ${stats_db_name}.publication_licenses
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.dataset_licenses
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.software_licenses
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_pids AS
|
||||||
|
select substr(o.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid
|
||||||
|
from ${openaire_db_name}.organization o lateral view explode(o.pid) pids as ppid;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_sources as
|
||||||
|
SELECT o.id, case when d.id is null then 'other' else o.datasource end as datasource
|
||||||
|
FROM (
|
||||||
|
SELECT substr(o.id, 4) as id, substr(instances.instance.key, 4) as datasource
|
||||||
|
from ${openaire_db_name}.organization o lateral view explode(o.collectedfrom) instances as instance) o
|
||||||
|
LEFT OUTER JOIN (
|
||||||
|
SELECT substr(d.id, 4) id
|
||||||
|
from ${openaire_db_name}.datasource d
|
||||||
|
WHERE d.datainfo.deletedbyinference=false) d on o.datasource = d.id;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Additional relations
|
||||||
|
--
|
||||||
|
-- Refereed related tables/views
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_refereed as
|
||||||
|
select substr(r.id, 4) as id, inst.refereed.classname as refereed
|
||||||
|
from ${openaire_db_name}.publication r lateral view explode(r.instance) instances as inst
|
||||||
|
where r.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_refereed as
|
||||||
|
select substr(r.id, 4) as id, inst.refereed.classname as refereed
|
||||||
|
from ${openaire_db_name}.dataset r lateral view explode(r.instance) instances as inst
|
||||||
|
where r.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_refereed as
|
||||||
|
select substr(r.id, 4) as id, inst.refereed.classname as refereed
|
||||||
|
from ${openaire_db_name}.software r lateral view explode(r.instance) instances as inst
|
||||||
|
where r.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_refereed as
|
||||||
|
select substr(r.id, 4) as id, inst.refereed.classname as refereed
|
||||||
|
from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
|
||||||
|
where r.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
|
||||||
|
select * from ${stats_db_name}.publication_refereed
|
||||||
|
union all
|
||||||
|
select * from ${stats_db_name}.dataset_refereed
|
||||||
|
union all
|
||||||
|
select * from ${stats_db_name}.software_refereed
|
||||||
|
union all
|
||||||
|
select * from ${stats_db_name}.otherresearchproduct_refereed;
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
----------------------------------------------------
|
||||||
|
-- Shortcuts for various definitions in stats db ---
|
||||||
|
----------------------------------------------------
|
||||||
|
|
||||||
|
-- Peer reviewed:
|
||||||
|
-- Results that have been collected from Crossref
|
||||||
|
create table ${stats_db_name}.result_peerreviewed as
|
||||||
|
with peer_reviewed as (
|
||||||
|
select distinct r.id as id
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
join ${stats_db_name}.result_sources rs on rs.id=r.id
|
||||||
|
join ${stats_db_name}.datasource d on d.id=rs.datasource
|
||||||
|
where d.name='Crossref')
|
||||||
|
select distinct peer_reviewed.id as id, true as peer_reviewed
|
||||||
|
from peer_reviewed
|
||||||
|
union all
|
||||||
|
select distinct r.id as id, false as peer_reviewed
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
left outer join peer_reviewed pr on pr.id=r.id
|
||||||
|
where pr.id is null;
|
||||||
|
|
||||||
|
-- Green OA:
|
||||||
|
-- OA results that are hosted by an Institutional repository and have NOT been harvested from a DOAJ journal.
|
||||||
|
create table ${stats_db_name}.result_greenoa as
|
||||||
|
with result_green as (
|
||||||
|
select distinct r.id as id
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
join ${stats_db_name}.result_datasources rd on rd.id=r.id
|
||||||
|
join ${stats_db_name}.datasource d on d.id=rd.datasource
|
||||||
|
left outer join (
|
||||||
|
select rd.id from ${stats_db_name}.result_datasources rd
|
||||||
|
join ${stats_db_name}.datasource d on rd.datasource=d.id
|
||||||
|
join ${stats_db_name}.datasource_sources sds on sds.id=d.id
|
||||||
|
join ${stats_db_name}.datasource sd on sd.id=sds.datasource
|
||||||
|
where sd.name='DOAJ-ARTICLES'
|
||||||
|
) as doaj on doaj.id=r.id
|
||||||
|
where r.bestlicence in ('Open Access', 'Open Source') and d.type='Institutional Repository' and doaj.id is null)
|
||||||
|
select distinct result_green.id, true as green
|
||||||
|
from result_green
|
||||||
|
union all
|
||||||
|
select distinct r.id as id, false as green
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
left outer join result_green rg on rg.id=r.id
|
||||||
|
where rg.id is null;
|
||||||
|
|
||||||
|
-- GOLD OA:
|
||||||
|
-- OA results that have been harvested from a DOAJ journal.
|
||||||
|
create table ${stats_db_name}.result_gold as
|
||||||
|
with result_gold as (
|
||||||
|
select distinct r.id as id
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
join ${stats_db_name}.result_datasources rd on rd.id=r.id
|
||||||
|
join ${stats_db_name}.datasource d on d.id=rd.datasource
|
||||||
|
join ${stats_db_name}.datasource_sources sds on sds.id=d.id
|
||||||
|
join ${stats_db_name}.datasource sd on sd.id=sds.datasource
|
||||||
|
where r.type='publication' and r.bestlicence='Open Access' and sd.name='DOAJ-Articles')
|
||||||
|
select distinct result_gold.id, true as gold
|
||||||
|
from result_gold
|
||||||
|
union all
|
||||||
|
select distinct r.id, false as gold
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
where r.id not in (select id from result_gold);
|
||||||
|
|
||||||
|
-- shortcut result-country through the organization affiliation
|
||||||
|
create table ${stats_db_name}.result_affiliated_country as
|
||||||
|
select r.id as id, o.country as country
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
join ${stats_db_name}.result_organization ro on ro.id=r.id
|
||||||
|
join ${stats_db_name}.organization o on o.id=ro.organization
|
||||||
|
where o.country is not null and o.country!='';
|
||||||
|
|
||||||
|
-- shortcut result-country through datasource of deposition
|
||||||
|
create table ${stats_db_name}.result_deposited_country as
|
||||||
|
select r.id as id, o.country as country
|
||||||
|
from ${stats_db_name}.result r
|
||||||
|
join ${stats_db_name}.result_datasources rd on rd.id=r.id
|
||||||
|
join ${stats_db_name}.datasource d on d.id=rd.datasource
|
||||||
|
join ${stats_db_name}.datasource_organizations dor on dor.id=d.id
|
||||||
|
join ${stats_db_name}.organization o on o.id=dor.organization
|
||||||
|
where o.country is not null and o.country!='';
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
-- replace the creation of the result view to include the boolean fields from the previous tables (green, gold,
|
||||||
|
-- peer reviewed)
|
||||||
|
drop table if exists ${stats_db_name}.result_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.result_tmp (
|
||||||
|
id STRING,
|
||||||
|
title STRING,
|
||||||
|
publisher STRING,
|
||||||
|
journal STRING,
|
||||||
|
`date` STRING,
|
||||||
|
`year` INT,
|
||||||
|
bestlicence STRING,
|
||||||
|
access_mode STRING,
|
||||||
|
embargo_end_date STRING,
|
||||||
|
delayed BOOLEAN,
|
||||||
|
authors INT,
|
||||||
|
source STRING,
|
||||||
|
abstract BOOLEAN,
|
||||||
|
type STRING ,
|
||||||
|
peer_reviewed BOOLEAN,
|
||||||
|
green BOOLEAN,
|
||||||
|
gold BOOLEAN)
|
||||||
|
clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
insert into ${stats_db_name}.result_tmp
|
||||||
|
select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
|
||||||
|
FROM ${stats_db_name}.publication r
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
|
||||||
|
|
||||||
|
insert into ${stats_db_name}.result_tmp
|
||||||
|
select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
|
||||||
|
FROM ${stats_db_name}.dataset r
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
|
||||||
|
|
||||||
|
insert into ${stats_db_name}.result_tmp
|
||||||
|
select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
|
||||||
|
FROM ${stats_db_name}.software r
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
|
||||||
|
|
||||||
|
insert into ${stats_db_name}.result_tmp
|
||||||
|
select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
|
||||||
|
FROM ${stats_db_name}.otherresearchproduct r
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
|
||||||
|
LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
|
||||||
|
|
||||||
|
drop table if exists ${stats_db_name}.result;
|
||||||
|
drop view if exists ${stats_db_name}.result;
|
||||||
|
create table ${stats_db_name}.result stored as parquet as select * from ${stats_db_name}.result_tmp;
|
||||||
|
drop table ${stats_db_name}.result_tmp;
|
||||||
|
|
@ -0,0 +1,163 @@
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Shadow schema table exchange
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
|
||||||
|
-- Dropping old views
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.country;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.countrygdp;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_citations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_classifications;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_concepts;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_licenses;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_pids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_sources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_topics;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_organizations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_results;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.fundref;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.numbers_country;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_projects;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_organizations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_citations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_classifications;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_concepts;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_licenses;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_pids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_sources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_topics;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_citations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_classifications;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_concepts;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_licenses;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_organization;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_pids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projects;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_sources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_topics;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.rndexpediture;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.roarmap;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_citations;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_classifications;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_concepts;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_datasources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_languages;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_licenses;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_oids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_pids;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_sources;
|
||||||
|
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_topics;
|
||||||
|
|
||||||
|
|
||||||
|
-- Creating the shadow database, in case it doesn't exist
|
||||||
|
CREATE database IF NOT EXISTS ${stats_db_shadow_name};
|
||||||
|
|
||||||
|
-- Creating new views
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.country AS SELECT * FROM ${stats_db_name}.country;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.countrygdp AS SELECT * FROM ${stats_db_name}.countrygdp;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset AS SELECT * FROM ${stats_db_name}.dataset;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_citations AS SELECT * FROM ${stats_db_name}.dataset_citations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_classifications AS SELECT * FROM ${stats_db_name}.dataset_classifications;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_concepts AS SELECT * FROM ${stats_db_name}.dataset_concepts;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_datasources AS SELECT * FROM ${stats_db_name}.dataset_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_languages AS SELECT * FROM ${stats_db_name}.dataset_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_licenses AS SELECT * FROM ${stats_db_name}.dataset_licenses;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_oids AS SELECT * FROM ${stats_db_name}.dataset_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_pids AS SELECT * FROM ${stats_db_name}.dataset_pids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_sources AS SELECT * FROM ${stats_db_name}.dataset_sources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_topics AS SELECT * FROM ${stats_db_name}.dataset_topics;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource AS SELECT * FROM ${stats_db_name}.datasource;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_languages AS SELECT * FROM ${stats_db_name}.datasource_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_oids AS SELECT * FROM ${stats_db_name}.datasource_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_organizations AS SELECT * FROM ${stats_db_name}.datasource_organizations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_results AS SELECT * FROM ${stats_db_name}.datasource_results;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.fundref AS SELECT * FROM ${stats_db_name}.fundref;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.numbers_country AS SELECT * FROM ${stats_db_name}.numbers_country;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization AS SELECT * FROM ${stats_db_name}.organization;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_datasources AS SELECT * FROM ${stats_db_name}.organization_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_projects AS SELECT * FROM ${stats_db_name}.organization_projects;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct AS SELECT * FROM ${stats_db_name}.otherresearchproduct;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations AS SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications AS SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts AS SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages AS SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses AS SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics AS SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project AS SELECT * FROM ${stats_db_name}.project;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_oids AS SELECT * FROM ${stats_db_name}.project_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_organizations AS SELECT * FROM ${stats_db_name}.project_organizations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results AS SELECT * FROM ${stats_db_name}.project_results;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication AS SELECT * FROM ${stats_db_name}.publication;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_citations AS SELECT * FROM ${stats_db_name}.publication_citations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_concepts AS SELECT * FROM ${stats_db_name}.publication_concepts;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_datasources AS SELECT * FROM ${stats_db_name}.publication_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_languages AS SELECT * FROM ${stats_db_name}.publication_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_licenses AS SELECT * FROM ${stats_db_name}.publication_licenses;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_oids AS SELECT * FROM ${stats_db_name}.publication_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_pids AS SELECT * FROM ${stats_db_name}.publication_pids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_sources AS SELECT * FROM ${stats_db_name}.publication_sources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_topics AS SELECT * FROM ${stats_db_name}.publication_topics;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result AS SELECT * FROM ${stats_db_name}.result;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_citations AS SELECT * FROM ${stats_db_name}.result_citations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_classifications AS SELECT * FROM ${stats_db_name}.result_classifications;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_concepts AS SELECT * FROM ${stats_db_name}.result_concepts;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_datasources AS SELECT * FROM ${stats_db_name}.result_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_languages AS SELECT * FROM ${stats_db_name}.result_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_licenses AS SELECT * FROM ${stats_db_name}.result_licenses;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_oids AS SELECT * FROM ${stats_db_name}.result_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_organization AS SELECT * FROM ${stats_db_name}.result_organization;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_pids AS SELECT * FROM ${stats_db_name}.result_pids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projects AS SELECT * FROM ${stats_db_name}.result_projects;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_sources AS SELECT * FROM ${stats_db_name}.result_sources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_topics AS SELECT * FROM ${stats_db_name}.result_topics;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.rndexpediture AS SELECT * FROM ${stats_db_name}.rndexpediture;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.roarmap AS SELECT * FROM ${stats_db_name}.roarmap;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software AS SELECT * FROM ${stats_db_name}.software;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_citations AS SELECT * FROM ${stats_db_name}.software_citations;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_classifications AS SELECT * FROM ${stats_db_name}.software_classifications;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_concepts AS SELECT * FROM ${stats_db_name}.software_concepts;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_datasources AS SELECT * FROM ${stats_db_name}.software_datasources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_languages AS SELECT * FROM ${stats_db_name}.software_languages;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_licenses AS SELECT * FROM ${stats_db_name}.software_licenses;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_oids AS SELECT * FROM ${stats_db_name}.software_oids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_pids AS SELECT * FROM ${stats_db_name}.software_pids;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_sources AS SELECT * FROM ${stats_db_name}.software_sources;
|
||||||
|
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_topics AS SELECT * FROM ${stats_db_name}.software_topics;
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Impala table statistics - Needed to make the tables
|
||||||
|
-- visible for impala
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
|
||||||
|
COMPUTE STATS country;
|
||||||
|
COMPUTE STATS countrygdp;
|
||||||
|
COMPUTE STATS dataset;
|
||||||
|
COMPUTE STATS dataset_citations;
|
||||||
|
COMPUTE STATS dataset_classifications;
|
||||||
|
COMPUTE STATS dataset_concepts;
|
||||||
|
COMPUTE STATS dataset_datasources;
|
||||||
|
COMPUTE STATS dataset_languages;
|
||||||
|
COMPUTE STATS dataset_oids;
|
||||||
|
COMPUTE STATS dataset_pids;
|
||||||
|
COMPUTE STATS dataset_sources;
|
||||||
|
COMPUTE STATS dataset_topics;
|
||||||
|
COMPUTE STATS datasource;
|
||||||
|
COMPUTE STATS datasource_languages;
|
||||||
|
COMPUTE STATS datasource_oids;
|
||||||
|
COMPUTE STATS datasource_organizations;
|
||||||
|
COMPUTE STATS datasource_results;
|
||||||
|
COMPUTE STATS fundref;
|
||||||
|
COMPUTE STATS numbers_country;
|
||||||
|
COMPUTE STATS organization;
|
||||||
|
COMPUTE STATS organization_datasources;
|
||||||
|
COMPUTE STATS organization_projects;
|
||||||
|
COMPUTE STATS otherresearchproduct;
|
||||||
|
COMPUTE STATS otherresearchproduct_citations;
|
||||||
|
COMPUTE STATS otherresearchproduct_classifications;
|
||||||
|
COMPUTE STATS otherresearchproduct_concepts;
|
||||||
|
COMPUTE STATS otherresearchproduct_datasources;
|
||||||
|
COMPUTE STATS otherresearchproduct_languages;
|
||||||
|
COMPUTE STATS otherresearchproduct_licenses;
|
||||||
|
COMPUTE STATS otherresearchproduct_oids;
|
||||||
|
COMPUTE STATS otherresearchproduct_pids;
|
||||||
|
COMPUTE STATS otherresearchproduct_sources;
|
||||||
|
COMPUTE STATS otherresearchproduct_topics;
|
||||||
|
COMPUTE STATS project;
|
||||||
|
COMPUTE STATS project_oids;
|
||||||
|
COMPUTE STATS project_organizations;
|
||||||
|
COMPUTE STATS project_results;
|
||||||
|
COMPUTE STATS publication;
|
||||||
|
COMPUTE STATS publication_citations;
|
||||||
|
COMPUTE STATS publication_classifications;
|
||||||
|
COMPUTE STATS publication_concepts;
|
||||||
|
COMPUTE STATS publication_datasources;
|
||||||
|
COMPUTE STATS publication_languages;
|
||||||
|
COMPUTE STATS publication_licenses;
|
||||||
|
COMPUTE STATS publication_oids;
|
||||||
|
COMPUTE STATS publication_pids;
|
||||||
|
COMPUTE STATS publication_sources;
|
||||||
|
COMPUTE STATS publication_topics;
|
||||||
|
COMPUTE STATS result;
|
||||||
|
COMPUTE STATS result_citations;
|
||||||
|
COMPUTE STATS result_classifications;
|
||||||
|
COMPUTE STATS result_concepts;
|
||||||
|
COMPUTE STATS result_datasources;
|
||||||
|
COMPUTE STATS result_languages;
|
||||||
|
COMPUTE STATS result_licenses;
|
||||||
|
COMPUTE STATS result_oids;
|
||||||
|
COMPUTE STATS result_organization;
|
||||||
|
COMPUTE STATS result_pids;
|
||||||
|
COMPUTE STATS result_projects;
|
||||||
|
COMPUTE STATS result_sources;
|
||||||
|
COMPUTE STATS result_topics;
|
||||||
|
COMPUTE STATS rndexpediture;
|
||||||
|
COMPUTE STATS roarmap;
|
||||||
|
COMPUTE STATS software;
|
||||||
|
COMPUTE STATS software_citations;
|
||||||
|
COMPUTE STATS software_classifications;
|
||||||
|
COMPUTE STATS software_concepts;
|
||||||
|
COMPUTE STATS software_datasources;
|
||||||
|
COMPUTE STATS software_languages;
|
||||||
|
COMPUTE STATS software_licenses;
|
||||||
|
COMPUTE STATS software_oids;
|
||||||
|
COMPUTE STATS software_pids;
|
||||||
|
COMPUTE STATS software_sources;
|
||||||
|
COMPUTE STATS software_topics;
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
--------------------------------------------------------------
|
||||||
|
--------------------------------------------------------------
|
||||||
|
-- Publication table/view and Publication related tables/views
|
||||||
|
--------------------------------------------------------------
|
||||||
|
--------------------------------------------------------------
|
||||||
|
|
||||||
|
-- Publication temporary table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.publication_tmp;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
INSERT INTO ${stats_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal ,
|
||||||
|
p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
|
||||||
|
p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
|
||||||
|
case when size(p.description) > 0 then true else false end as abstract,
|
||||||
|
'publication' as type
|
||||||
|
from ${openaire_db_name}.publication p
|
||||||
|
where p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN (SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language FROM ${openaire_db_name}.publication p;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_topics as select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||||
|
|
||||||
|
-- Publication_citations
|
||||||
|
CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
--------------------------------------------------------------
|
|
||||||
--------------------------------------------------------------
|
|
||||||
-- 2. Publication table/view and Publication related tables/views
|
|
||||||
--------------------------------------------------------------
|
|
||||||
--------------------------------------------------------------
|
|
||||||
|
|
||||||
-- Publication temporary table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.publication_tmp;
|
|
||||||
|
|
||||||
CREATE TABLE ${stats_db_name}.publication_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
-- The following throws the following exception on CRN HUE Hive:
|
|
||||||
-- Error while compiling statement: FAILED: SemanticException [Error 10011]: Line 2:34 Invalid function 'date_format'
|
|
||||||
-- But runs OK on OCEAN HUE Hive
|
|
||||||
|
|
||||||
INSERT INTO ${stats_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal ,
|
|
||||||
p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
|
|
||||||
p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
|
|
||||||
case when size(p.description) > 0 then true else false end as abstract,
|
|
||||||
'publication' as type
|
|
||||||
from ${openaire_db_name}.publication p
|
|
||||||
where p.datainfo.deletedbyinference=false;
|
|
||||||
|
|
||||||
-- INSERT INTO ${hive_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal,
|
|
||||||
-- p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
|
|
||||||
-- p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
|
|
||||||
-- case when size(p.description) > 0 then true else false end as abstract,
|
|
||||||
-- 'publication' as type
|
|
||||||
-- from openaire.publication p
|
|
||||||
-- where p.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.publication_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN (SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.publication p;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids as ids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.publication_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.publication p lateral view explode(p.pid) pids as ppid;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.publication_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.publication p lateral view explode(p.subject) subjects as subject;
|
|
||||||
|
|
@ -1,2 +1,36 @@
|
||||||
-- 3. Publication_citations
|
------------------------------------------------------
|
||||||
CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
------------------------------------------------------
|
||||||
|
-- Dataset table/view and Dataset related tables/views
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
|
||||||
|
-- Dataset temporary table supporting updates
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.dataset_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored AS orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
INSERT INTO ${stats_db_name}.dataset_tmp SELECT substr(d.id, 4) AS id, d.title[0].value AS title, d.publisher.value AS publisher, cast(null AS string) AS journal,
|
||||||
|
d.dateofacceptance.value as date, date_format(d.dateofacceptance.value,'yyyy') AS year, d.bestaccessright.classname AS bestlicence,
|
||||||
|
d.embargoenddate.value AS embargo_end_date, false AS delayed, size(d.author) AS authors , concat_ws('\u003B',d.source.value) AS source,
|
||||||
|
CASE WHEN SIZE(d.description) > 0 THEN TRUE ELSE FALSE end AS abstract,
|
||||||
|
'dataset' AS type
|
||||||
|
FROM ${openaire_db_name}.dataset d
|
||||||
|
WHERE d.datainfo.deletedbyinference=FALSE;
|
||||||
|
|
||||||
|
-- Dataset_citations
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_citations AS SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.dataset d LATERAL VIEW explode(d.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_datasources AS SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||||
|
FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
|
||||||
|
(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.dataset p;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.dataset_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
--------------------------------------------------------
|
||||||
|
--------------------------------------------------------
|
||||||
|
-- Software table/view and Software related tables/views
|
||||||
|
--------------------------------------------------------
|
||||||
|
--------------------------------------------------------
|
||||||
|
|
||||||
|
-- Software temporary table supporting updates
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.software_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.software_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
INSERT INTO ${stats_db_name}.software_tmp SELECT substr(s.id, 4) as id, s.title[0].value AS title, s.publisher.value AS publisher, CAST(NULL AS string) AS journal,
|
||||||
|
s.dateofacceptance.value AS DATE, date_format(s.dateofacceptance.value,'yyyy') AS YEAR, s.bestaccessright.classname AS bestlicence,
|
||||||
|
s.embargoenddate.value AS embargo_end_date, FALSE AS delayed, SIZE(s.author) AS authors , concat_ws('\u003B',s.source.value) AS source,
|
||||||
|
CASE WHEN SIZE(s.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
|
||||||
|
'software' as type
|
||||||
|
from ${openaire_db_name}.software s
|
||||||
|
where s.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
-- Software_citations
|
||||||
|
CREATE TABLE ${stats_db_name}.software_citations AS SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.software s LATERAL VIEW explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts AS context;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||||
|
FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
|
||||||
|
(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.software p;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.software_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
------------------------------------------------------
|
|
||||||
------------------------------------------------------
|
|
||||||
-- 4. Dataset table/view and Dataset related tables/views
|
|
||||||
------------------------------------------------------
|
|
||||||
------------------------------------------------------
|
|
||||||
|
|
||||||
-- Dataset temporary table supporting updates
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.dataset_tmp;
|
|
||||||
CREATE TABLE ${stats_db_name}.dataset_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.dataset_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.dataset p lateral view explode(p.subject) subjects as subject;
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
INSERT INTO ${stats_db_name}.dataset_tmp select substr(d.id, 4) as id, d.title[0].value as title, d.publisher.value as publisher, cast(null as string) as journal,
|
|
||||||
d.dateofacceptance.value as date, date_format(d.dateofacceptance.value,'yyyy') as year, d.bestaccessright.classname as bestlicence,
|
|
||||||
d.embargoenddate.value as embargo_end_date, false as delayed, size(d.author) as authors , concat_ws('\u003B',d.source.value) as source,
|
|
||||||
case when size(d.description) > 0 then true else false end as abstract,
|
|
||||||
'dataset' as type
|
|
||||||
from ${openaire_db_name}.dataset d
|
|
||||||
where d.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Dataset_citations
|
|
||||||
Create table ${stats_db_name}.dataset_citations as select substr(d.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.dataset d lateral view explode(d.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context;
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.dataset_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
|
|
||||||
from ${openaire_db_name}.dataset p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
|
|
||||||
(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.dataset_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.dataset p;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids as ids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.dataset_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.dataset p lateral view explode(p.pid) pids as ppid;
|
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
-- Otherresearchproduct table/view and Otherresearchproduct related tables/views
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
-- Otherresearchproduct temporary table supporting updates
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_tmp ( id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
INSERT INTO ${stats_db_name}.otherresearchproduct_tmp SELECT substr(o.id, 4) AS id, o.title[0].value AS title, o.publisher.value AS publisher, CAST(NULL AS string) AS journal,
|
||||||
|
o.dateofacceptance.value AS DATE, date_format(o.dateofacceptance.value,'yyyy') AS year, o.bestaccessright.classname AS bestlicence,
|
||||||
|
o.embargoenddate.value as embargo_end_date, FALSE AS delayed, SIZE(o.author) AS authors , concat_ws('\u003B',o.source.value) AS source,
|
||||||
|
CASE WHEN SIZE(o.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
|
||||||
|
'other' AS type
|
||||||
|
FROM ${openaire_db_name}.otherresearchproduct o
|
||||||
|
WHERE o.datainfo.deletedbyinference=FALSE;
|
||||||
|
|
||||||
|
-- Otherresearchproduct_citations
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context;
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
|
||||||
|
from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
|
||||||
|
(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.otherresearchproduct p;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid;
|
||||||
|
|
||||||
|
CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject;
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
--------------------------------------------------------
|
|
||||||
--------------------------------------------------------
|
|
||||||
-- 5. Software table/view and Software related tables/views
|
|
||||||
--------------------------------------------------------
|
|
||||||
--------------------------------------------------------
|
|
||||||
|
|
||||||
-- Software temporary table supporting updates
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.software_tmp;
|
|
||||||
CREATE TABLE ${stats_db_name}.software_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.software_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.software p lateral view explode(p.subject) subjects as subject;
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
INSERT INTO ${stats_db_name}.software_tmp select substr(s.id, 4) as id, s.title[0].value as title, s.publisher.value as publisher, cast(null as string) as journal,
|
|
||||||
s.dateofacceptance.value as date, date_format(s.dateofacceptance.value,'yyyy') as year, s.bestaccessright.classname as bestlicence,
|
|
||||||
s.embargoenddate.value as embargo_end_date, false as delayed, size(s.author) as authors , concat_ws('\u003B',s.source.value) as source,
|
|
||||||
case when size(s.description) > 0 then true else false end as abstract,
|
|
||||||
'software' as type
|
|
||||||
from ${openaire_db_name}.software s
|
|
||||||
where s.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Software_citations
|
|
||||||
Create table ${stats_db_name}.software_citations as select substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.software s lateral view explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts as context;
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.software_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
|
|
||||||
from ${openaire_db_name}.software p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
|
|
||||||
(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.software p;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids as ids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.software_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.software p lateral view explode(p.pid) pids as ppid;
|
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
-- noinspection SqlNoDataSourceInspectionForFile
|
||||||
|
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Project table/view and Project related tables/views
|
||||||
|
------------------------------------------------------
|
||||||
|
------------------------------------------------------
|
||||||
|
-- Project_oids Table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.project_oids;
|
||||||
|
CREATE TABLE ${stats_db_name}.project_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids;
|
||||||
|
|
||||||
|
-- Project_organizations Table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.project_organizations;
|
||||||
|
CREATE TABLE ${stats_db_name}.project_organizations AS SELECT substr(r.source, 4) AS id, substr(r.target, 4) AS organization from ${openaire_db_name}.relation r WHERE r.reltype='projectOrganization';
|
||||||
|
|
||||||
|
-- Project_results Table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.project_results;
|
||||||
|
CREATE TABLE ${stats_db_name}.project_results AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result FROM ${openaire_db_name}.relation r WHERE r.reltype='resultProject' and r.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
-- Project table
|
||||||
|
----------------
|
||||||
|
-- Creating and populating temporary Project table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.project_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.project_tmp (id STRING, acronym STRING, title STRING, funder STRING, funding_lvl0 STRING, funding_lvl1 STRING, funding_lvl2 STRING, ec39 STRING, type STRING, startdate STRING, enddate STRING, start_year INT, end_year INT, duration INT, haspubs STRING, numpubs INT, daysforlastpub INT, delayedpubs INT, callidentifier STRING, code STRING) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
INSERT INTO ${stats_db_name}.project_tmp SELECT substr(p.id, 4) AS id, p.acronym.value AS acronym, p.title.value AS title, xpath_string(p.fundingtree[0].value, '//funder/name') AS funder, xpath_string(p.fundingtree[0].value, '//funding_level_0/name') AS funding_lvl0, xpath_string(p.fundingtree[0].value, '//funding_level_1/name') AS funding_lvl1, xpath_string(p.fundingtree[0].value, '//funding_level_2/name') AS funding_lvl2, p.ecsc39.value AS ec39, p.contracttype.classname AS type, p.startdate.value AS startdate, p.enddate.value AS enddate, year(p.startdate.value) AS start_year, year(p.enddate.value) AS end_year, CAST(MONTHS_BETWEEN(p.enddate.value, p.startdate.value) AS INT) AS duration, 'no' AS haspubs, 0 AS numpubs, 0 AS daysforlastpub, 0 AS delayedpubs, p.callidentifier.value AS callidentifier, p.code.value AS code FROM ${openaire_db_name}.project p WHERE p.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
create table ${stats_db_name}.funder as
|
||||||
|
select distinct xpath_string(fund, '//funder/id') as id, xpath_string(fund, '//funder/name') as name, xpath_string(fund, '//funder/shortname') as shortname
|
||||||
|
from ${openaire_db_name}.project p lateral view explode(p.fundingtree.value) fundingtree as fund
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
--------------------------------------------------------------------------------
|
|
||||||
--------------------------------------------------------------------------------
|
|
||||||
-- 6. Otherresearchproduct table/view and Otherresearchproduct related tables/views
|
|
||||||
--------------------------------------------------------------------------------
|
|
||||||
--------------------------------------------------------------------------------
|
|
||||||
|
|
||||||
-- Otherresearchproduct temporary table supporting updates
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_tmp;
|
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_tmp ( id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN, authors INT, source STRING, abstract BOOLEAN, type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.otherresearchproduct_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.subject) subjects as subject;
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
INSERT INTO ${stats_db_name}.otherresearchproduct_tmp select substr(o.id, 4) as id, o.title[0].value as title, o.publisher.value as publisher, cast(null as string) as journal,
|
|
||||||
o.dateofacceptance.value as date, date_format(o.dateofacceptance.value,'yyyy') as year, o.bestaccessright.classname as bestlicence,
|
|
||||||
o.embargoenddate.value as embargo_end_date, false as delayed, size(o.author) as authors , concat_ws('\u003B',o.source.value) as source,
|
|
||||||
case when size(o.description) > 0 then true else false end as abstract,
|
|
||||||
'other' as type
|
|
||||||
from ${openaire_db_name}.otherresearchproduct o
|
|
||||||
where o.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Otherresearchproduct_citations
|
|
||||||
Create table ${stats_db_name}.otherresearchproduct_citations as select substr(o.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.otherresearchproduct o lateral view explode(o.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts as context;
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
|
|
||||||
from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
|
|
||||||
(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.otherresearchproduct p;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids as ids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
create table ${stats_db_name}.otherresearchproduct_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.pid) pids as ppid;
|
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
----------------------------------------------------
|
||||||
|
----------------------------------------------------
|
||||||
|
-- Result table/view and Result related tables/views
|
||||||
|
----------------------------------------------------
|
||||||
|
----------------------------------------------------
|
||||||
|
|
||||||
|
-- Views on temporary tables that should be re-created in the end
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.publication_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.software_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.dataset_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.otherresearchproduct_tmp;
|
||||||
|
|
||||||
|
-- Views on final tables
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_datasources AS SELECT * FROM ${stats_db_name}.publication_datasources UNION ALL SELECT * FROM ${stats_db_name}.software_datasources UNION ALL SELECT * FROM ${stats_db_name}.dataset_datasources UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_citations AS SELECT * FROM ${stats_db_name}.publication_citations UNION ALL SELECT * FROM ${stats_db_name}.software_citations UNION ALL SELECT * FROM ${stats_db_name}.dataset_citations UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications UNION ALL SELECT * FROM ${stats_db_name}.software_classifications UNION ALL SELECT * FROM ${stats_db_name}.dataset_classifications UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_concepts AS SELECT * FROM ${stats_db_name}.publication_concepts UNION ALL SELECT * FROM ${stats_db_name}.software_concepts UNION ALL SELECT * FROM ${stats_db_name}.dataset_concepts UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_languages AS SELECT * FROM ${stats_db_name}.publication_languages UNION ALL SELECT * FROM ${stats_db_name}.software_languages UNION ALL SELECT * FROM ${stats_db_name}.dataset_languages UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_oids AS SELECT * FROM ${stats_db_name}.publication_oids UNION ALL SELECT * FROM ${stats_db_name}.software_oids UNION ALL SELECT * FROM ${stats_db_name}.dataset_oids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_pids AS SELECT * FROM ${stats_db_name}.publication_pids UNION ALL SELECT * FROM ${stats_db_name}.software_pids UNION ALL SELECT * FROM ${stats_db_name}.dataset_pids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.result_topics AS SELECT * FROM ${stats_db_name}.publication_topics UNION ALL SELECT * FROM ${stats_db_name}.software_topics UNION ALL SELECT * FROM ${stats_db_name}.dataset_topics UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.result_organization;
|
||||||
|
CREATE TABLE ${stats_db_name}.result_organization AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r WHERE r.reltype='resultOrganization';
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.result_projects;
|
||||||
|
CREATE TABLE ${stats_db_name}.result_projects AS select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend FROM ${stats_db_name}.result r JOIN ${stats_db_name}.project_results pr ON r.id=pr.result JOIN ${stats_db_name}.project_tmp p ON p.id=pr.id;
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
------------------------------------------------------
|
|
||||||
------------------------------------------------------
|
|
||||||
-- 7. Project table/view and Project related tables/views
|
|
||||||
------------------------------------------------------
|
|
||||||
------------------------------------------------------
|
|
||||||
-- Project_oids Table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.project_oids;
|
|
||||||
CREATE TABLE ${stats_db_name}.project_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids as ids;
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
-- Project_organizations Table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.project_organizations;
|
|
||||||
CREATE TABLE ${stats_db_name}.project_organizations AS SELECT substr(r.source, 4) as id, substr(r.target, 4) AS organization from ${openaire_db_name}.relation r WHERE r.reltype='projectOrganization';
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
-- Project_results Table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.project_results;
|
|
||||||
CREATE TABLE ${stats_db_name}.project_results AS SELECT substr(r.target, 4) as id, substr(r.source, 4) AS result from ${openaire_db_name}.relation r WHERE r.reltype='resultProject';
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
-- Project table
|
|
||||||
----------------
|
|
||||||
-- Creating and populating temporary Project table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.project_tmp;
|
|
||||||
CREATE TABLE ${stats_db_name}.project_tmp (id STRING, acronym STRING, title STRING, funder STRING, funding_lvl0 STRING, funding_lvl1 STRING, funding_lvl2 STRING, ec39 STRING, type STRING, startdate STRING, enddate STRING, start_year STRING, end_year STRING, duration INT, haspubs STRING, numpubs INT, daysforlastpub INT, delayedpubs INT, callidentifier STRING, code STRING) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
INSERT INTO ${stats_db_name}.project_tmp SELECT substr(p.id, 4) as id, p.acronym.value as acronym, p.title.value as title, xpath_string(p.fundingtree[0].value, '//funder/name') as funder, xpath_string(p.fundingtree[0].value, '//funding_level_0/name') as funding_lvl0, xpath_string(p.fundingtree[0].value, '//funding_level_1/name') as funding_lvl1, xpath_string(p.fundingtree[0].value, '//funding_level_2/name') as funding_lvl2, p.ecsc39.value as ec39, p.contracttype.classname as type, p.startdate.value as startdate, p.enddate.value as enddate, date_format(p.startdate.value, 'yyyy') as start_year, date_format(p.enddate.value, 'yyyy') as end_year, 0 as duration, 'no' as haspubs, 0 as numpubs, 0 as daysforlastpub, 0 as delayedpubs, p.callidentifier.value as callidentifier, p.code.value as code from ${openaire_db_name}.project p WHERE p.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
-- noinspection SqlNoDataSourceInspectionForFile
|
||||||
|
|
||||||
|
------------------------------------------------------------
|
||||||
|
------------------------------------------------------------
|
||||||
|
-- Datasource table/view and Datasource related tables/views
|
||||||
|
------------------------------------------------------------
|
||||||
|
------------------------------------------------------------
|
||||||
|
|
||||||
|
-- Datasource table creation & update
|
||||||
|
-------------------------------------
|
||||||
|
-- Creating and populating temporary datasource table
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.datasource_tmp;
|
||||||
|
CREATE TABLE ${stats_db_name}.datasource_tmp(`id` string, `name` STRING, `type` STRING, `dateofvalidation` STRING, `yearofvalidation` string, `harvested` BOOLEAN, `piwik_id` INT, `latitude` STRING, `longitude`STRING, `websiteurl` STRING, `compatibility` STRING) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
|
||||||
|
|
||||||
|
-- Insert statement that takes into account the piwik_id of the openAIRE graph
|
||||||
|
INSERT INTO ${stats_db_name}.datasource_tmp
|
||||||
|
SELECT substr(d1.id, 4) AS id, officialname.value AS name,
|
||||||
|
datasourcetype.classname AS type, dateofvalidation.value AS dateofvalidation, date_format(d1.dateofvalidation.value,'yyyy') AS yearofvalidation,
|
||||||
|
FALSE AS harvested,
|
||||||
|
CASE WHEN d2.piwik_id IS NULL THEN 0 ELSE d2.piwik_id END AS piwik_id,
|
||||||
|
d1.latitude.value AS latitude, d1.longitude.value AS longitude,
|
||||||
|
d1.websiteurl.value AS websiteurl, d1.openairecompatibility.classid AS compatibility
|
||||||
|
FROM ${openaire_db_name}.datasource d1
|
||||||
|
LEFT OUTER JOIN
|
||||||
|
(SELECT id, split(originalidd, '\\:')[1] as piwik_id
|
||||||
|
FROM ${openaire_db_name}.datasource
|
||||||
|
LATERAL VIEW EXPLODE(originalid) temp AS originalidd
|
||||||
|
WHERE originalidd like "piwik:%") AS d2
|
||||||
|
ON d1.id = d2.id
|
||||||
|
WHERE d1.datainfo.deletedbyinference=FALSE;
|
||||||
|
|
||||||
|
-- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table.
|
||||||
|
-- Creating a temporary dual table that will be removed after the following insert
|
||||||
|
CREATE TABLE ${stats_db_name}.dual(dummy CHAR(1));
|
||||||
|
INSERT INTO ${stats_db_name}.dual VALUES('X');
|
||||||
|
INSERT INTO ${stats_db_name}.datasource_tmp (`id`, `name`, `type`, `dateofvalidation`, `yearofvalidation`, `harvested`, `piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`)
|
||||||
|
SELECT 'other', 'Other', 'Repository', NULL, NULL, false, 0, NULL, NULL, NULL, 'unknown' FROM ${stats_db_name}.dual WHERE 'other' not in (SELECT id FROM ${stats_db_name}.datasource_tmp WHERE name='Unknown Repository');
|
||||||
|
DROP TABLE ${stats_db_name}.dual;
|
||||||
|
|
||||||
|
UPDATE ${stats_db_name}.datasource_tmp SET name='Other' WHERE name='Unknown Repository';
|
||||||
|
UPDATE ${stats_db_name}.datasource_tmp SET yearofvalidation=null WHERE yearofvalidation='-1';
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.datasource_languages;
|
||||||
|
CREATE TABLE ${stats_db_name}.datasource_languages AS SELECT substr(d.id, 4) AS id, langs.languages AS language FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs AS languages;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.datasource_oids;
|
||||||
|
CREATE TABLE ${stats_db_name}.datasource_oids AS SELECT substr(d.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids AS ids;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.datasource_organizations;
|
||||||
|
CREATE TABLE ${stats_db_name}.datasource_organizations AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r WHERE r.reltype='datasourceOrganization';
|
||||||
|
|
||||||
|
-- datasource sources:
|
||||||
|
-- where the datasource info have been collected from.
|
||||||
|
create table if not exists ${stats_db_name}.datasource_sources AS select substr(d.id,4) as id, substr(cf.key, 4) as datasource from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfrom as cf where d.datainfo.deletedbyinference=false;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS SELECT datasource AS id, id AS result FROM ${stats_db_name}.result_datasources;
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
----------------------------------------------------
|
|
||||||
----------------------------------------------------
|
|
||||||
-- 8. Result table/view and Result related tables/views
|
|
||||||
----------------------------------------------------
|
|
||||||
----------------------------------------------------
|
|
||||||
|
|
||||||
-- Views on temporary tables that should be re-created in the end
|
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence as access_mode FROM ${stats_db_name}.publication_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.software_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.dataset_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.otherresearchproduct_tmp;
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.result_organization;
|
|
||||||
CREATE TABLE ${stats_db_name}.result_organization AS SELECT substr(r.target, 4) as id, substr(r.source, 4) as organization from ${openaire_db_name}.relation r where r.reltype='resultOrganization';
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.result_projects;
|
|
||||||
CREATE TABLE ${stats_db_name}.result_projects AS select pr.result as id, pr.id as project, datediff(p.enddate, p.startdate) as daysfromend from ${stats_db_name}.result r join ${stats_db_name}.project_results pr on r.id=pr.result join ${stats_db_name}.project_tmp p on p.id=pr.id;
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
-- Views on final tables
|
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_datasources as SELECT * FROM ${stats_db_name}.publication_datasources UNION ALL SELECT * FROM ${stats_db_name}.software_datasources UNION ALL SELECT * FROM ${stats_db_name}.dataset_datasources UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_citations as SELECT * FROM ${stats_db_name}.publication_citations UNION ALL SELECT * FROM ${stats_db_name}.software_citations UNION ALL SELECT * FROM ${stats_db_name}.dataset_citations UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_classifications as SELECT * FROM ${stats_db_name}.publication_classifications UNION ALL SELECT * FROM ${stats_db_name}.software_classifications UNION ALL SELECT * FROM ${stats_db_name}.dataset_classifications UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_concepts as SELECT * FROM ${stats_db_name}.publication_concepts UNION ALL SELECT * FROM ${stats_db_name}.software_concepts UNION ALL SELECT * FROM ${stats_db_name}.dataset_concepts UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_languages as SELECT * FROM ${stats_db_name}.publication_languages UNION ALL SELECT * FROM ${stats_db_name}.software_languages UNION ALL SELECT * FROM ${stats_db_name}.dataset_languages UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_oids as SELECT * FROM ${stats_db_name}.publication_oids UNION ALL SELECT * FROM ${stats_db_name}.software_oids UNION ALL SELECT * FROM ${stats_db_name}.dataset_oids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_pids as SELECT * FROM ${stats_db_name}.publication_pids UNION ALL SELECT * FROM ${stats_db_name}.software_pids UNION ALL SELECT * FROM ${stats_db_name}.dataset_pids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
CREATE OR REPLACE VIEW ${stats_db_name}.result_topics as SELECT * FROM ${stats_db_name}.publication_topics UNION ALL SELECT * FROM ${stats_db_name}.software_topics UNION ALL SELECT * FROM ${stats_db_name}.dataset_topics UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
----------------------------------------------------------------
|
||||||
|
----------------------------------------------------------------
|
||||||
|
-- Organization table/view and Organization related tables/views
|
||||||
|
----------------------------------------------------------------
|
||||||
|
----------------------------------------------------------------
|
||||||
|
DROP TABLE IF EXISTS ${stats_db_name}.organization;
|
||||||
|
CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization AS SELECT substr(o.id, 4) as id, o.legalname.value as name, o.legalshortname.value as legalshortname, o.country.classid as country
|
||||||
|
FROM ${openaire_db_name}.organization o WHERE o.datainfo.deletedbyinference=FALSE;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS SELECT organization AS id, id AS datasource FROM ${stats_db_name}.datasource_organizations;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW ${stats_db_name}.organization_projects AS SELECT id AS project, organization as id FROM ${stats_db_name}.project_organizations;
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
------------------------------------------------------------
|
|
||||||
------------------------------------------------------------
|
|
||||||
-- 9. Datasource table/view and Datasource related tables/views
|
|
||||||
------------------------------------------------------------
|
|
||||||
------------------------------------------------------------
|
|
||||||
-- Datasource table creation & update
|
|
||||||
-------------------------------------
|
|
||||||
-- Creating and populating temporary datasource table
|
|
||||||
DROP TABLE IF EXISTS ${stats_db_name}.datasource_tmp;
|
|
||||||
create table ${stats_db_name}.datasource_tmp(`id` string, `name` string, `type` string, `dateofvalidation` string, `yearofvalidation` string, `harvested` boolean, `piwik_id` int, `latitude` string, `longitude` string, `websiteurl` string, `compatibility` string) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
insert into ${stats_db_name}.datasource_tmp SELECT substr(d.id, 4) as id, officialname.value as name, datasourcetype.classname as type, dateofvalidation.value as dateofvalidation, date_format(d.dateofvalidation.value,'yyyy') as yearofvalidation, false as harvested, 0 as piwik_id, d.latitude.value as latitude, d.longitude.value as longitude, d.websiteurl.value as websiteurl, d.openairecompatibility.classid as compatibility
|
|
||||||
from ${openaire_db_name}.datasource d
|
|
||||||
WHERE d.datainfo.deletedbyinference=false;
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
-- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table. To see if default values are there
|
|
||||||
-- Creating a temporary dual table that will be removed after the following insert
|
|
||||||
CREATE TABLE ${stats_db_name}.dual(dummy char(1));
|
|
||||||
INSERT INTO ${stats_db_name}.dual values('X');
|
|
||||||
INSERT INTO ${stats_db_name}.datasource_tmp (`id`, `name`, `type`, `dateofvalidation`, `yearofvalidation`, `harvested`, `piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`)
|
|
||||||
SELECT 'other', 'Other', 'Repository', NULL, NULL, false, 0, NULL, NULL, NULL, 'unknown' FROM ${stats_db_name}.dual WHERE 'other' not in (SELECT id FROM ${stats_db_name}.datasource_tmp WHERE name='Unknown Repository');
|
|
||||||
DROP TABLE ${stats_db_name}.dual;
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue