D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-applications

This commit is contained in:
Enrico Ottonello 2022-06-24 11:52:31 +02:00
parent a70a327281 7511ef94c2
commit 3f61e6fce2
21 changed files with 137 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/bioschemas-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-broker-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-broker-public-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-mdstore-manager/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dnet-exporter-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

27
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java
View File

@ -27,9 +27,6 @@ public class DnetOpenaireExporterProperties {
private ClassPathResource findContextProfilesByType;
private ClassPathResource getRepoProfile;
@Value("${openaire.exporter.contentLoadQuery}")
private String contentLoadQuery;
private String isLookupUrl;
private String objectStoreServiceUrl;
private String isRegistryServiceUrl;
@ -53,6 +50,7 @@ public class DnetOpenaireExporterProperties {
private Vocabularies vocabularies;
public static class Datasource {
// MONGODB
private String mongoHost;
private int mongoPort;
@ -244,6 +242,7 @@ public class DnetOpenaireExporterProperties {
}
public static class Swagger {
private String apiTitle;
private String apiDescription;
private String apiLicense;
@ -390,7 +389,7 @@ public class DnetOpenaireExporterProperties {
this.findContextProfiles = findContextProfiles;
}
public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) {
public void setFindContextProfilesByType(final ClassPathResource findContextProfilesByType) {
this.findContextProfilesByType = findContextProfilesByType;
}
@ -406,14 +405,6 @@ public class DnetOpenaireExporterProperties {
this.getRepoProfile = getRepoProfile;
}
public String getContentLoadQuery() {
return contentLoadQuery;
}
public void setContentLoadQuery(String contentLoadQuery) {
this.contentLoadQuery = contentLoadQuery;
}
public String getIsLookupUrl() {
return isLookupUrl;
}
@ -458,7 +449,7 @@ public class DnetOpenaireExporterProperties {
return cxfClientConnectTimeout;
}
public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) {
public void setCxfClientConnectTimeout(final int cxfClientConnectTimeout) {
this.cxfClientConnectTimeout = cxfClientConnectTimeout;
}
@ -466,7 +457,7 @@ public class DnetOpenaireExporterProperties {
return cxfClientReceiveTimeout;
}
public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) {
public void setCxfClientReceiveTimeout(final int cxfClientReceiveTimeout) {
this.cxfClientReceiveTimeout = cxfClientReceiveTimeout;
}
@ -534,9 +525,13 @@ public class DnetOpenaireExporterProperties {
this.swaggerContexts = swaggerContexts;
}
public Swagger getSwaggerInfo() { return swaggerInfo; }
public Swagger getSwaggerInfo() {
return swaggerInfo;
}
public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; }
public void setSwaggerInfo(final Swagger swaggerInfo) {
this.swaggerInfo = swaggerInfo;
}
public Vocabularies getVocabularies() {
return vocabularies;

141
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java
View File

@ -1,7 +1,13 @@
package eu.dnetlib.openaire.dsm.dao;
import java.io.IOException;
import static com.mongodb.client.model.Filters.and;
import static com.mongodb.client.model.Filters.eq;
import static com.mongodb.client.model.Filters.regex;
import static com.mongodb.client.model.Projections.fields;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -9,25 +15,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.cache.*;
import com.google.common.primitives.Ints;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.AggregationStage;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.miscutils.datetime.DateUtils;
import eu.dnetlib.openaire.common.Utils;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
import eu.dnetlib.openaire.dsm.domain.CollectionMode;
import eu.dnetlib.openaire.dsm.domain.TransformationInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.logging.Log;
@ -42,8 +29,28 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import static com.mongodb.client.model.Filters.*;
import static com.mongodb.client.model.Projections.fields;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.primitives.Ints;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.AggregationStage;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.miscutils.datetime.DateUtils;
import eu.dnetlib.openaire.common.Utils;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
import eu.dnetlib.openaire.dsm.domain.CollectionMode;
import eu.dnetlib.openaire.dsm.domain.TransformationInfo;
import eu.dnetlib.openaire.info.JdbcInfoDao;
/**
* Created by claudio on 20/10/2016.
*/
@ -59,19 +66,23 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
@Autowired
private DnetOpenaireExporterProperties config;
@Autowired
private JdbcInfoDao jdbcInfoDao;
private final static String LOADTIME = "loadtime";
private final LoadingCache<String, Instant> loadingCache = CacheBuilder.newBuilder()
.maximumSize(1)
.expireAfterWrite(60, TimeUnit.MINUTES)
.build(new CacheLoader<String, Instant>() {
// The only cached value is associated to "loadtime"
public Instant load(String key) {
final Instant loadTime = getLoadTime();
log.debug("found load time: " + loadTime.toString());
return loadTime;
}
});
.maximumSize(1)
.expireAfterWrite(60, TimeUnit.MINUTES)
.build(new CacheLoader<String, Instant>() {
// The only cached value is associated to "loadtime"
@Override
public Instant load(final String key) {
final Instant loadTime = getLoadTime();
log.debug("found load time: " + loadTime.toString());
return loadTime;
}
});
private static final Bson fields = getFields();
@ -84,14 +95,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
final Datasource conf = config.getDatasource();
try {
final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory(dsId, "(collect|transform)"))
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
.map(getMapper())
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
.map(getMapper())
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
final Instant loadTime = loadingCache.get(LOADTIME);
@ -105,18 +116,15 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
}
return aggregationInfos;
} catch (Throwable e) {
} catch (final Throwable e) {
throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
}
}
private Instant getLoadTime() {
log.warn("querying for metadata load time, not using cache");
return Optional.ofNullable(getCollection().find(queryForLastMetadataLoad()))
.map(d -> d.sort(dbo("system:startHumanDate", -1)).first())
.map(d -> (String) d.getOrDefault("system:startHumanDate", ""))
.map(s -> Instant.parse(s.replaceAll("\\+.*", "Z")))
.orElse(Instant.MIN);
final LocalDate date = jdbcInfoDao.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date);
return date.atStartOfDay(ZoneId.systemDefault()).toInstant();
}
private Instant asInstant(final AggregationInfo a) {
@ -124,7 +132,9 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
}
@Override
@CacheEvict(cacheNames = { "dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache" }, allEntries = true)
@CacheEvict(cacheNames = {
"dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache"
}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped dsManager aggregation history cache");
@ -141,7 +151,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
switch (stage) {
case COLLECT:
CollectionInfo cInfo = new CollectionInfo();
final CollectionInfo cInfo = new CollectionInfo();
cInfo.setAggregationStage(stage);
cInfo.setCollectionMode(getCollectionMode(d));
cInfo.setNumberOfRecords(getNumberOfRecords(d));
@ -149,7 +159,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
info = cInfo;
break;
case TRANSFORM:
TransformationInfo tInfo = new TransformationInfo();
final TransformationInfo tInfo = new TransformationInfo();
tInfo.setAggregationStage(stage);
tInfo.setNumberOfRecords(getNumberOfRecords(d));
tInfo.setDate(getDate(d));
@ -159,13 +169,12 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
return info;
}
private CollectionMode getCollectionMode(Document d) {
private CollectionMode getCollectionMode(final Document d) {
return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
.map(CollectionMode::valueOf)
.orElseGet(() ->
Optional.ofNullable(d.getString("collectionMode"))
.map(CollectionMode::valueOf)
.orElse(null));
.map(CollectionMode::valueOf)
.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
.map(CollectionMode::valueOf)
.orElse(null));
}
private Integer getNumberOfRecords(final Document d) {
@ -190,15 +199,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
}
private static Bson getFields() {
return fields(
eq("system:wfName", 1),
eq("system:node:SELECT_MODE:selection", 1),
eq("collectionMode", 1),
eq("mainlog:sinkSize", 1),
eq("mainlog:writeOps", 1),
eq("mainlog:total", 1),
eq("system:startHumanDate", 1),
eq("system:profileName", 1));
return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1));
}
private static BasicDBObject dbo(final String key, final Object value) {
@ -206,21 +207,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
}
private Bson queryForAggregationHistory(final String dsId, final String pattern) {
return and(
eq("parentDatasourceId", dsId),
eq("system:profileFamily", "aggregator"),
eq("system:isCompletedSuccessfully", "true"),
regex("system:wfName", pattern, "i"));
}
private Bson queryForLastMetadataLoad() {
try {
final String contentLoadQuery = config.getContentLoadQuery();
log.debug("parsing content load query: " + contentLoadQuery);
return new ObjectMapper().readValue(contentLoadQuery, BasicDBObject.class);
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), eq("system:isCompletedSuccessfully", "true"), regex("system:wfName", pattern, "i"));
}
private synchronized MongoCollection<Document> getCollection() {

1
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java
View File

@ -80,6 +80,7 @@ public class JdbcApiDaoImpl implements JdbcApiDao {
res.put("FP7", "ec__________::EC::FP7");
res.put("H2020", "ec__________::EC::H2020");
res.put("HE", "ec__________::EC::HE");
log.debug(String.format("processed %s funding ids", res.size()));
res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v)));
return res;

2
apps/dnet-exporter-api/src/main/resources/global.properties
View File

@ -48,8 +48,6 @@ openaire.exporter.findContextProfiles = /eu/dnetlib/openaire/xquery/find
openaire.exporter.findContextProfilesByType = /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
openaire.exporter.getRepoProfile = /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
openaire.exporter.contentLoadQuery = { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
# REST API CONFIGURATION
openaire.exporter.swaggerDsm.apiTitle = OpenAIRE aggregator REST API
openaire.exporter.swaggerDsm.apiDescription = The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.

2
apps/dnet-orgs-database-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/scholexplorer-api/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

37
apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/HomeController.java Normal file
View File

@ -0,0 +1,37 @@
package eu.dnetlib.scholix.api.controller;
import eu.dnetlib.common.controller.AbstractDnetController;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
@Controller
public class HomeController extends AbstractDnetController {
@GetMapping({
"/doc", "/swagger"
})
public String apiDoc() {
return "redirect:swagger-ui/";
}
@GetMapping({
"/v1/ui"
})
public String v1Doc() {
return "redirect:/swagger-ui/?urls.primaryName=Scholexplorer%20API%20V1.0";
}
@GetMapping({
"/v2/ui"
})
public String v2Doc() {
return "redirect:/swagger-ui/?urls.primaryName=Scholexplorer%20API%20V2.0";
}
}

15
apps/scholexplorer-api/src/main/resources/application.properties
View File

@ -6,13 +6,15 @@ dhp.swagger.api.host = localhost:8080
dhp.swagger.api.basePath = /
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
#
#spring.thymeleaf.cache=false
#
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
management.endpoints.web.exposure.include = prometheus,health
management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
management.endpoint.health.show-details = always
management.metrics.distribution.percentiles-histogram.http.server.requests=false
management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
@ -20,7 +22,8 @@ management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95,
scholix.elastic.clusterNodes = localhost:9200
scholix.elastic.indexName = dli_shadow_scholix
scholix.elastic.clusterNodes = 10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200
#scholix.elastic.clusterNodes = localhost:9200
scholix.elastic.indexName = dli_scholix
scholix.elastic.socketTimeout = 60000
scholix.elastic.connectionTimeout= 60000

2
cmd-line-apps/dhp-broker-client/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>cmd-line-apps</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
cmd-line-apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-broker-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-openaire-broker-common/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
pom.xml
View File

@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.8-SNAPSHOT</version>
<version>3.2.9-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>