Merge branch 'master' of code-repo.d4science.org:D-Net/dnet-applications

apps/bioschemas-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.2.8-SNAPSHOT</version>
+        <version>3.2.9-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

apps/dhp-broker-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-broker-public-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dhp-mdstore-manager/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/dnet-exporter-api/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java

@@ -27,9 +27,6 @@ public class DnetOpenaireExporterProperties {
 	private ClassPathResource findContextProfilesByType;
 	private ClassPathResource getRepoProfile;
 
-	@Value("${openaire.exporter.contentLoadQuery}")
-	private String contentLoadQuery;
-
 	private String isLookupUrl;
 	private String objectStoreServiceUrl;
 	private String isRegistryServiceUrl;
@@ -53,6 +50,7 @@ public class DnetOpenaireExporterProperties {
 	private Vocabularies vocabularies;
 
 	public static class Datasource {
+
 		// MONGODB
 		private String mongoHost;
 		private int mongoPort;
@@ -244,6 +242,7 @@ public class DnetOpenaireExporterProperties {
 	}
 
 	public static class Swagger {
+
 		private String apiTitle;
 		private String apiDescription;
 		private String apiLicense;
@@ -390,7 +389,7 @@ public class DnetOpenaireExporterProperties {
 		this.findContextProfiles = findContextProfiles;
 	}
 
-	public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) {
+	public void setFindContextProfilesByType(final ClassPathResource findContextProfilesByType) {
 		this.findContextProfilesByType = findContextProfilesByType;
 	}
 
@@ -406,14 +405,6 @@ public class DnetOpenaireExporterProperties {
 		this.getRepoProfile = getRepoProfile;
 	}
 
-	public String getContentLoadQuery() {
-		return contentLoadQuery;
-	}
-
-	public void setContentLoadQuery(String contentLoadQuery) {
-		this.contentLoadQuery = contentLoadQuery;
-	}
-
 	public String getIsLookupUrl() {
 		return isLookupUrl;
 	}
@@ -458,7 +449,7 @@ public class DnetOpenaireExporterProperties {
 		return cxfClientConnectTimeout;
 	}
 
-	public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) {
+	public void setCxfClientConnectTimeout(final int cxfClientConnectTimeout) {
 		this.cxfClientConnectTimeout = cxfClientConnectTimeout;
 	}
 
@@ -466,7 +457,7 @@ public class DnetOpenaireExporterProperties {
 		return cxfClientReceiveTimeout;
 	}
 
-	public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) {
+	public void setCxfClientReceiveTimeout(final int cxfClientReceiveTimeout) {
 		this.cxfClientReceiveTimeout = cxfClientReceiveTimeout;
 	}
 
@@ -534,9 +525,13 @@ public class DnetOpenaireExporterProperties {
 		this.swaggerContexts = swaggerContexts;
 	}
 
-	public Swagger getSwaggerInfo() { return swaggerInfo; }
+	public Swagger getSwaggerInfo() {
+		return swaggerInfo;
+	}
 
-	public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; }
+	public void setSwaggerInfo(final Swagger swaggerInfo) {
+		this.swaggerInfo = swaggerInfo;
+	}
 
 	public Vocabularies getVocabularies() {
 		return vocabularies;

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -1,7 +1,13 @@
 package eu.dnetlib.openaire.dsm.dao;
 
-import java.io.IOException;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.regex;
+import static com.mongodb.client.model.Projections.fields;
+
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -9,25 +15,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.cache.*;
-import com.google.common.primitives.Ints;
-import com.mongodb.BasicDBObject;
-import com.mongodb.MongoClient;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.MongoCollection;
-import eu.dnetlib.DnetOpenaireExporterProperties;
-import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
-import eu.dnetlib.enabling.datasources.common.AggregationStage;
-import eu.dnetlib.enabling.datasources.common.DsmException;
-import eu.dnetlib.miscutils.datetime.DateUtils;
-import eu.dnetlib.openaire.common.Utils;
-
-import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
-import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
-import eu.dnetlib.openaire.dsm.domain.CollectionMode;
-import eu.dnetlib.openaire.dsm.domain.TransformationInfo;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateFormatUtils;
 import org.apache.commons.logging.Log;
@@ -42,8 +29,28 @@ import org.springframework.cache.annotation.Cacheable;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
-import static com.mongodb.client.model.Filters.*;
+import com.google.common.cache.CacheBuilder;
-import static com.mongodb.client.model.Projections.fields;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.primitives.Ints;
+import com.mongodb.BasicDBObject;
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+
+import eu.dnetlib.DnetOpenaireExporterProperties;
+import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
+import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.enabling.datasources.common.AggregationStage;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.openaire.common.Utils;
+import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
+import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
+import eu.dnetlib.openaire.dsm.domain.CollectionMode;
+import eu.dnetlib.openaire.dsm.domain.TransformationInfo;
+import eu.dnetlib.openaire.info.JdbcInfoDao;
+
 /**
  * Created by claudio on 20/10/2016.
  */
@@ -59,20 +66,24 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	@Autowired
 	private DnetOpenaireExporterProperties config;
 
+	@Autowired
+	private JdbcInfoDao jdbcInfoDao;
+
 	private final static String LOADTIME = "loadtime";
 	private final LoadingCache<String, Instant> loadingCache = CacheBuilder.newBuilder()
 		.maximumSize(1)
 		.expireAfterWrite(60, TimeUnit.MINUTES)
 		.build(new CacheLoader<String, Instant>() {
+
 			// The only cached value is associated to "loadtime"
-					public Instant load(String key) {
+			@Override
+			public Instant load(final String key) {
 				final Instant loadTime = getLoadTime();
 				log.debug("found load time: " + loadTime.toString());
 				return loadTime;
 			}
 		});
 
-
 	private static final Bson fields = getFields();
 
 	private static MongoCollection<Document> collection = null;
@@ -105,18 +116,15 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			}
 
 			return aggregationInfos;
-		} catch (Throwable e) {
+		} catch (final Throwable e) {
 			throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
 		}
 	}
 
 	private Instant getLoadTime() {
 		log.warn("querying for metadata load time, not using cache");
-		return Optional.ofNullable(getCollection().find(queryForLastMetadataLoad()))
+		final LocalDate date = jdbcInfoDao.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date);
-				.map(d -> d.sort(dbo("system:startHumanDate", -1)).first())
+		return date.atStartOfDay(ZoneId.systemDefault()).toInstant();
-				.map(d -> (String) d.getOrDefault("system:startHumanDate", ""))
-				.map(s -> Instant.parse(s.replaceAll("\\+.*", "Z")))
-				.orElse(Instant.MIN);
 	}
 
 	private Instant asInstant(final AggregationInfo a) {
@@ -124,7 +132,9 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	}
 
 	@Override
-	@CacheEvict(cacheNames = { "dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache" }, allEntries = true)
+	@CacheEvict(cacheNames = {
+		"dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache"
+	}, allEntries = true)
 	@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
 	public void dropCache() {
 		log.debug("dropped dsManager aggregation history cache");
@@ -141,7 +151,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				switch (stage) {
 
 				case COLLECT:
-					CollectionInfo cInfo = new CollectionInfo();
+					final CollectionInfo cInfo = new CollectionInfo();
 					cInfo.setAggregationStage(stage);
 					cInfo.setCollectionMode(getCollectionMode(d));
 					cInfo.setNumberOfRecords(getNumberOfRecords(d));
@@ -149,7 +159,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 					info = cInfo;
 					break;
 				case TRANSFORM:
-					TransformationInfo tInfo = new TransformationInfo();
+					final TransformationInfo tInfo = new TransformationInfo();
 					tInfo.setAggregationStage(stage);
 					tInfo.setNumberOfRecords(getNumberOfRecords(d));
 					tInfo.setDate(getDate(d));
@@ -159,11 +169,10 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				return info;
 			}
 
-			private CollectionMode getCollectionMode(Document d) {
+			private CollectionMode getCollectionMode(final Document d) {
 				return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
 					.map(CollectionMode::valueOf)
-										.orElseGet(() ->
+					.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
-												Optional.ofNullable(d.getString("collectionMode"))
 						.map(CollectionMode::valueOf)
 						.orElse(null));
 			}
@@ -190,15 +199,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	}
 
 	private static Bson getFields() {
-		return fields(
+		return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1));
-				eq("system:wfName", 1),
-				eq("system:node:SELECT_MODE:selection", 1),
-				eq("collectionMode", 1),
-				eq("mainlog:sinkSize", 1),
-				eq("mainlog:writeOps", 1),
-				eq("mainlog:total", 1),
-				eq("system:startHumanDate", 1),
-				eq("system:profileName", 1));
 	}
 
 	private static BasicDBObject dbo(final String key, final Object value) {
@@ -206,21 +207,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	}
 
 	private Bson queryForAggregationHistory(final String dsId, final String pattern) {
-		return and(
+		return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), eq("system:isCompletedSuccessfully", "true"), regex("system:wfName", pattern, "i"));
-				eq("parentDatasourceId", dsId),
-				eq("system:profileFamily", "aggregator"),
-				eq("system:isCompletedSuccessfully", "true"),
-				regex("system:wfName", pattern, "i"));
-	}
-
-	private Bson queryForLastMetadataLoad() {
-		try {
-			final String contentLoadQuery = config.getContentLoadQuery();
-			log.debug("parsing content load query: " + contentLoadQuery);
-			return new ObjectMapper().readValue(contentLoadQuery, BasicDBObject.class);
-		} catch (IOException e) {
-			throw new IllegalArgumentException(e);
-		}
 	}
 
 	private synchronized MongoCollection<Document> getCollection() {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java

@@ -80,6 +80,7 @@ public class JdbcApiDaoImpl implements JdbcApiDao {
 
 		res.put("FP7", "ec__________::EC::FP7");
 		res.put("H2020", "ec__________::EC::H2020");
+		res.put("HE", "ec__________::EC::HE");
 		log.debug(String.format("processed %s funding ids", res.size()));
 		res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v)));
 		return res;

apps/dnet-exporter-api/src/main/resources/global.properties

@@ -48,8 +48,6 @@ openaire.exporter.findContextProfiles       =   /eu/dnetlib/openaire/xquery/find
 openaire.exporter.findContextProfilesByType =   /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery
 openaire.exporter.getRepoProfile            =   /eu/dnetlib/openaire/xquery/getRepoProfile.xquery
 
-openaire.exporter.contentLoadQuery          =   { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] }
-
 # REST API CONFIGURATION
 openaire.exporter.swaggerDsm.apiTitle          =   OpenAIRE aggregator REST API
 openaire.exporter.swaggerDsm.apiDescription    =   The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically.

apps/dnet-orgs-database-application/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../pom.xml</relativePath>
 	</parent>
 

apps/scholexplorer-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.dnetlib.dhp</groupId>
         <artifactId>apps</artifactId>
-        <version>3.2.8-SNAPSHOT</version>
+        <version>3.2.9-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

apps/scholexplorer-api/src/main/java/eu/dnetlib/scholix/api/controller/HomeController.java

@@ -0,0 +1,37 @@
+package eu.dnetlib.scholix.api.controller;
+
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+
+@Controller
+public class HomeController extends AbstractDnetController {
+
+    @GetMapping({
+            "/doc", "/swagger"
+    })
+    public String apiDoc() {
+        return "redirect:swagger-ui/";
+    }
+
+    @GetMapping({
+            "/v1/ui"
+    })
+    public String v1Doc() {
+        return "redirect:/swagger-ui/?urls.primaryName=Scholexplorer%20API%20V1.0";
+    }
+
+
+    @GetMapping({
+            "/v2/ui"
+    })
+    public String v2Doc() {
+        return "redirect:/swagger-ui/?urls.primaryName=Scholexplorer%20API%20V2.0";
+    }
+
+
+
+
+
+}

apps/scholexplorer-api/src/main/resources/application.properties

@@ -6,6 +6,7 @@ dhp.swagger.api.host = localhost:8080
 dhp.swagger.api.basePath = /
 
 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/scholexplorer-api/effective-pom.xml
+
 #
 #spring.thymeleaf.cache=false
 #
@@ -13,6 +14,7 @@ management.endpoints.web.exposure.include        = prometheus,health
 management.endpoints.web.base-path                  = /
 management.endpoints.web.path-mapping.prometheus    = metrics
 management.endpoints.web.path-mapping.health        = health
+management.endpoint.health.show-details             = always
 
 management.metrics.distribution.percentiles-histogram.http.server.requests=false
 management.metrics.distribution.slo.http.server.requests=50ms, 100ms, 200ms, 400ms
@@ -20,7 +22,8 @@ management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95,
 
 
 
-scholix.elastic.clusterNodes     =  localhost:9200
+scholix.elastic.clusterNodes     =  10.19.65.51:9200,10.19.65.52:9200,10.19.65.53:9200,10.19.65.54:9200
-scholix.elastic.indexName        =  dli_shadow_scholix
+#scholix.elastic.clusterNodes     =  localhost:9200
+scholix.elastic.indexName        =  dli_scholix
 scholix.elastic.socketTimeout    =  60000
 scholix.elastic.connectionTimeout=  60000

cmd-line-apps/dhp-broker-client/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>cmd-line-apps</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

cmd-line-apps/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-broker-apps-common/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/dnet-openaire-broker-common/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>libs</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

libs/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
 		<artifactId>dnet-applications</artifactId>
-		<version>3.2.8-SNAPSHOT</version>
+		<version>3.2.9-SNAPSHOT</version>
 		<relativePath>../</relativePath>
 	</parent>
 

pom.xml

@@ -10,7 +10,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>eu.dnetlib.dhp</groupId>
 	<artifactId>dnet-applications</artifactId>
-	<version>3.2.8-SNAPSHOT</version>
+	<version>3.2.9-SNAPSHOT</version>
 	<packaging>pom</packaging>
 
 	<licenses>