diff --git a/apps/dnet-exporter-api/README.md b/apps/dnet-exporter-api/README.md
new file mode 100644
index 00000000..a0a8d3f5
--- /dev/null
+++ b/apps/dnet-exporter-api/README.md
@@ -0,0 +1,6 @@
+SpringBoot application implementing OpenAIRE REST API to manage
+- Datasources
+- Contexts
+- Communities
+- Funders
+- Projects
\ No newline at end of file
diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml
new file mode 100644
index 00000000..9d1433d6
--- /dev/null
+++ b/apps/dnet-exporter-api/pom.xml
@@ -0,0 +1,149 @@
+
+
+
+ eu.dnetlib.dhp
+ apps
+ 3.2.4-SNAPSHOT
+ ../
+
+
+ 4.0.0
+ dnet-exporter-api
+ jar
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter-jdbc
+
+
+ org.springframework.boot
+ spring-boot-starter-data-jpa
+
+
+ org.springframework.boot
+ spring-boot-starter-cache
+
+
+ eu.dnetlib
+ cnr-rmi-api
+ [2.0.0,3.0.0)
+
+
+ org.apache.cxf
+ cxf-rt-transports-http
+ 3.1.5
+
+
+ eu.dnetlib
+ cnr-service-common
+ [2.0.0,3.0.0)
+
+
+ eu.dnetlib
+ dnet-openaireplus-mapping-utils
+ [6.3.0,7.0.0)
+
+
+ com.sun.jersey
+ jersey-client
+
+
+ eu.dnetlib
+ dnet-hadoop-commons
+
+
+
+
+ eu.dnetlib
+ dnet-objectstore-rmi
+ [2.0.0,3.0.0)
+
+
+ org.apache.solr
+ solr-solrj
+ 7.5.0
+
+
+
+ org.postgresql
+ postgresql
+
+
+
+ org.apache.commons
+ commons-dbcp2
+
+
+
+ org.antlr
+ stringtemplate
+ 3.2.1
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ org.apache.httpcomponents
+ httpmime
+
+
+ net.sf.supercsv
+ super-csv
+ 2.4.0
+
+
+ com.google.code.gson
+ gson
+
+
+
+ com.fasterxml.jackson.datatype
+ jackson-datatype-joda
+
+
+ joda-time
+ joda-time
+ 2.8.2
+
+
+
+ org.mongodb
+ mongo-java-driver
+ 3.4.2
+
+
+
+ eu.dnetlib
+ dnet-datasource-manager-common
+ [1.1.0,2.0.0)
+
+
+
+ org.springframework.boot
+ spring-boot-starter-test
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-help-plugin
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+
+
+
+
+
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java
new file mode 100644
index 00000000..cac25806
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java
@@ -0,0 +1,29 @@
+package eu.dnetlib;
+
+import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer;
+import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
+import org.springframework.stereotype.Component;
+
+import static java.util.Arrays.asList;
+
+@Component
+public class CacheCustomizer implements CacheManagerCustomizer {
+
+ @Override
+ public void customize(final ConcurrentMapCacheManager cacheManager) {
+ cacheManager.setCacheNames(
+ asList(
+ "fundingpath-ids",
+ "indexdsinfo-cache",
+ "objectstoreid-cache",
+ "context-cache",
+ "context-cache-funder",
+ "context-cache-community",
+ "dsm-aggregationhistory-cache",
+ "dsm-firstharvestdate-cache",
+ "vocabularies-cache",
+ "community-cache",
+ "info"));
+ }
+
+}
\ No newline at end of file
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java
new file mode 100644
index 00000000..1152802e
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java
@@ -0,0 +1,31 @@
+package eu.dnetlib;
+
+import java.io.IOException;
+import javax.servlet.http.HttpServletResponse;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@EnableCaching
+@RestController
+@EnableScheduling
+@SpringBootApplication
+@EnableAutoConfiguration(exclude = { SolrAutoConfiguration.class })
+public class DNetOpenaireExporterApplication {
+
+ @RequestMapping(value = { "/", "/docs" })
+ public void index(final HttpServletResponse response) throws IOException {
+ response.sendRedirect("swagger-ui.html");
+ }
+
+ public static void main(String[] args) throws Exception {
+ SpringApplication.run(DNetOpenaireExporterApplication.class, args);
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java
new file mode 100644
index 00000000..32e1940b
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterBeanFactory.java
@@ -0,0 +1,141 @@
+package eu.dnetlib;
+
+import com.google.common.collect.Lists;
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
+import com.mongodb.ServerAddress;
+import eu.dnetlib.OpenaireExporterConfig.Jdbc;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import io.micrometer.core.instrument.ImmutableTag;
+import io.micrometer.core.instrument.Metrics;
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.cxf.endpoint.Client;
+import org.apache.cxf.frontend.ClientProxy;
+import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
+import org.apache.cxf.transport.http.HTTPConduit;
+import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
+import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.ResourceLoader;
+
+import javax.annotation.PostConstruct;
+import javax.sql.DataSource;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+/**
+ * Created by claudio on 07/07/2017.
+ */
+@Configuration
+public class DNetOpenaireExporterBeanFactory {
+
+ private static final Log log = LogFactory.getLog(DNetOpenaireExporterBeanFactory.class);
+
+ @Autowired
+ private ResourceLoader resourceLoader;
+
+ @Value("pom.xml")
+ private ClassPathResource pom;
+
+ @PostConstruct
+ public void init() {
+ final MavenXpp3Reader reader = new MavenXpp3Reader();
+ try {
+ final Model model = reader.read(new InputStreamReader(pom.getInputStream()));
+
+ log.info(String.format("registering metric for %s", model.getArtifactId()));
+ Metrics.gauge("micrometer_info", Lists.newArrayList(
+ new ImmutableTag("component", model.getGroupId()+":"+model.getArtifactId()),
+ new ImmutableTag("version", model.getVersion()),
+ new ImmutableTag("scmtag", model.getScm().getTag())), 1);
+ } catch (IOException | XmlPullParserException e) {
+ log.error(e);
+ }
+ }
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ @Bean
+ public ISLookUpService getLookUpService() {
+ return getServiceStub(ISLookUpService.class, config.getIsLookupUrl());
+ }
+
+ @Bean
+ public ObjectStoreService getObjectStoreService() {
+ return getServiceStub(ObjectStoreService.class, config.getObjectStoreServiceUrl());
+ }
+
+ @Bean
+ public ISRegistryService getRegistryService() {
+ return getServiceStub(ISRegistryService.class, config.getIsRegistryServiceUrl());
+ }
+
+ @SuppressWarnings("unchecked")
+ private T getServiceStub(final Class clazz, final String endpoint) {
+ log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(),endpoint));
+ final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
+ jaxWsProxyFactory.setServiceClass(clazz);
+ jaxWsProxyFactory.setAddress(endpoint);
+
+ final T service = (T) jaxWsProxyFactory.create();
+
+ Client client = ClientProxy.getClient(service);
+ if (client != null) {
+ HTTPConduit conduit = (HTTPConduit) client.getConduit();
+ HTTPClientPolicy policy = new HTTPClientPolicy();
+
+ log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s",
+ config.getCxfClientConnectTimeout(),
+ config.getCxfClientReceiveTimeout(),
+ clazz.getCanonicalName()));
+
+ policy.setConnectionTimeout(config.getCxfClientConnectTimeout());
+ policy.setReceiveTimeout(config.getCxfClientReceiveTimeout());
+ conduit.setClient(policy);
+ }
+
+ return service;
+ }
+
+ @Bean
+ public DataSource getSqlDataSource() {
+ final Jdbc jdbc = config.getJdbc();
+ return getDatasource(
+ jdbc.getDriverClassName(),
+ jdbc.getUrl(),
+ jdbc.getUser(),
+ jdbc.getPwd(),
+ jdbc.getMinIdle(),
+ jdbc.getMaxRows());
+ }
+
+ private BasicDataSource getDatasource(String driverClassName, String jdbcUrl, String jdbcUser, String jdbcPwd, int jdbcMinIdle, int jdbcMaxIdle) {
+ final BasicDataSource d = new BasicDataSource();
+ d.setDriverClassName(driverClassName);
+ d.setUrl(jdbcUrl);
+ d.setUsername(jdbcUser);
+ d.setPassword(jdbcPwd);
+ d.setMinIdle(jdbcMinIdle);
+ d.setMaxIdle(jdbcMaxIdle);
+ return d;
+ }
+
+ @Bean
+ public MongoClient getMongoClient() {
+ return new MongoClient(
+ new ServerAddress(config.getDatasource().getMongoHost(), config.getDatasource().getMongoPort()),
+ MongoClientOptions.builder().connectionsPerHost(config.getDatasource().getMongoConnectionsPerHost()).build());
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java
new file mode 100644
index 00000000..87a54dd4
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/OpenaireExporterConfig.java
@@ -0,0 +1,548 @@
+package eu.dnetlib;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * Created by Alessia Bardi on 31/03/17.
+ *
+ * @author Alessia Bardi, Claudio Atzori
+ */
+@Configuration
+@PropertySource("classpath:global.properties")
+@ConfigurationProperties(prefix = "openaire.exporter")
+public class OpenaireExporterConfig {
+
+ // ISLOOKUP
+ private ClassPathResource findSolrIndexUrl;
+ private ClassPathResource findIndexDsInfo;
+ private ClassPathResource findObjectStore;
+ private ClassPathResource findFunderContexts;
+ private ClassPathResource findCommunityContexts;
+ private ClassPathResource findContextProfiles;
+ private ClassPathResource findContextProfilesByType;
+ private ClassPathResource getRepoProfile;
+
+ @Value("${openaire.exporter.contentLoadQuery}")
+ private String contentLoadQuery;
+
+ private String isLookupUrl;
+ private String objectStoreServiceUrl;
+ private String isRegistryServiceUrl;
+
+ private int requestWorkers = 100;
+ private int requestTimeout = 10;
+
+ private int cxfClientConnectTimeout = 120;
+ private int cxfClientReceiveTimeout = 120;
+
+ private Datasource datasource;
+ private Project project;
+ private Jdbc jdbc;
+ private Swagger swaggerDsm;
+ private Swagger swaggerProjects;
+ private Swagger swaggerFunders;
+ private Swagger swaggerCommunities;
+ private Swagger swaggerContexts;
+ private Swagger swaggerInfo;
+
+ private Vocabularies vocabularies;
+
+ public static class Datasource {
+ // MONGODB
+ private String mongoHost;
+ private int mongoPort;
+ private String mongoCollectionName;
+ private String mongoDbName;
+ private int mongoConnectionsPerHost;
+ private int mongoQueryLimit;
+
+ public String getMongoHost() {
+ return mongoHost;
+ }
+
+ public void setMongoHost(final String mongoHost) {
+ this.mongoHost = mongoHost;
+ }
+
+ public int getMongoPort() {
+ return mongoPort;
+ }
+
+ public void setMongoPort(final int mongoPort) {
+ this.mongoPort = mongoPort;
+ }
+
+ public String getMongoCollectionName() {
+ return mongoCollectionName;
+ }
+
+ public void setMongoCollectionName(final String mongoCollectionName) {
+ this.mongoCollectionName = mongoCollectionName;
+ }
+
+ public String getMongoDbName() {
+ return mongoDbName;
+ }
+
+ public void setMongoDbName(final String mongoDbName) {
+ this.mongoDbName = mongoDbName;
+ }
+
+ public int getMongoConnectionsPerHost() {
+ return mongoConnectionsPerHost;
+ }
+
+ public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) {
+ this.mongoConnectionsPerHost = mongoConnectionsPerHost;
+ }
+
+ public int getMongoQueryLimit() {
+ return mongoQueryLimit;
+ }
+
+ public void setMongoQueryLimit(final int mongoQueryLimit) {
+ this.mongoQueryLimit = mongoQueryLimit;
+ }
+ }
+
+ public static class Project {
+
+ private int flushSize;
+ private String tsvFields;
+ private Resource projectsFundingQueryTemplate;
+ private Resource dspaceTemplate;
+ private Resource dspaceHeadTemplate;
+ private Resource dspaceTailTemplate;
+ private Resource eprintsTemplate;
+
+ public int getFlushSize() {
+ return flushSize;
+ }
+
+ public void setFlushSize(final int flushSize) {
+ this.flushSize = flushSize;
+ }
+
+ public String getTsvFields() {
+ return tsvFields;
+ }
+
+ public void setTsvFields(final String tsvFields) {
+ this.tsvFields = tsvFields;
+ }
+
+ public Resource getProjectsFundingQueryTemplate() {
+ return projectsFundingQueryTemplate;
+ }
+
+ public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) {
+ this.projectsFundingQueryTemplate = projectsFundingQueryTemplate;
+ }
+
+ public Resource getDspaceTemplate() {
+ return dspaceTemplate;
+ }
+
+ public void setDspaceTemplate(final Resource dspaceTemplate) {
+ this.dspaceTemplate = dspaceTemplate;
+ }
+
+ public Resource getDspaceHeadTemplate() {
+ return dspaceHeadTemplate;
+ }
+
+ public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) {
+ this.dspaceHeadTemplate = dspaceHeadTemplate;
+ }
+
+ public Resource getDspaceTailTemplate() {
+ return dspaceTailTemplate;
+ }
+
+ public void setDspaceTailTemplate(final Resource dspaceTailTemplate) {
+ this.dspaceTailTemplate = dspaceTailTemplate;
+ }
+
+ public Resource getEprintsTemplate() {
+ return eprintsTemplate;
+ }
+
+ public void setEprintsTemplate(final Resource eprintsTemplate) {
+ this.eprintsTemplate = eprintsTemplate;
+ }
+ }
+
+ public static class Jdbc {
+
+ // JDBC
+ @Value("${spring.datasource.driverClassName}")
+ private String driverClassName;
+
+ private String url;
+ private String user;
+ private String pwd;
+ private int minIdle;
+ private int maxidle;
+ private int maxRows;
+
+ public String getDriverClassName() {
+ return driverClassName;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(final String url) {
+ this.url = url;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(final String user) {
+ this.user = user;
+ }
+
+ public String getPwd() {
+ return pwd;
+ }
+
+ public void setPwd(final String pwd) {
+ this.pwd = pwd;
+ }
+
+ public int getMinIdle() {
+ return minIdle;
+ }
+
+ public void setMinIdle(final int minIdle) {
+ this.minIdle = minIdle;
+ }
+
+ public int getMaxidle() {
+ return maxidle;
+ }
+
+ public void setMaxidle(final int maxidle) {
+ this.maxidle = maxidle;
+ }
+
+ public int getMaxRows() {
+ return maxRows;
+ }
+
+ public void setMaxRows(final int maxRows) {
+ this.maxRows = maxRows;
+ }
+ }
+
+ public static class Swagger {
+ private String apiTitle;
+ private String apiDescription;
+ private String apiLicense;
+ private String apiLicenseUrl;
+ private String apiContactName;
+ private String apiContactUrl;
+ private String apiContactEmail;
+
+ public String getApiTitle() {
+ return apiTitle;
+ }
+
+ public void setApiTitle(final String apiTitle) {
+ this.apiTitle = apiTitle;
+ }
+
+ public String getApiDescription() {
+ return apiDescription;
+ }
+
+ public void setApiDescription(final String apiDescription) {
+ this.apiDescription = apiDescription;
+ }
+
+ public String getApiLicense() {
+ return apiLicense;
+ }
+
+ public void setApiLicense(final String apiLicense) {
+ this.apiLicense = apiLicense;
+ }
+
+ public String getApiLicenseUrl() {
+ return apiLicenseUrl;
+ }
+
+ public void setApiLicenseUrl(final String apiLicenseUrl) {
+ this.apiLicenseUrl = apiLicenseUrl;
+ }
+
+ public String getApiContactName() {
+ return apiContactName;
+ }
+
+ public void setApiContactName(final String apiContactName) {
+ this.apiContactName = apiContactName;
+ }
+
+ public String getApiContactUrl() {
+ return apiContactUrl;
+ }
+
+ public void setApiContactUrl(final String apiContactUrl) {
+ this.apiContactUrl = apiContactUrl;
+ }
+
+ public String getApiContactEmail() {
+ return apiContactEmail;
+ }
+
+ public void setApiContactEmail(final String apiContactEmail) {
+ this.apiContactEmail = apiContactEmail;
+ }
+ }
+
+ public static class Vocabularies {
+
+ private String baseUrl;
+
+ private String countriesEndpoint;
+
+ private String datasourceTypologiesEndpoint;
+
+ public String getCountriesEndpoint() {
+ return countriesEndpoint;
+ }
+
+ public void setCountriesEndpoint(final String countriesEndpoint) {
+ this.countriesEndpoint = countriesEndpoint;
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setBaseUrl(final String baseUrl) {
+ this.baseUrl = baseUrl;
+ }
+
+ public String getDatasourceTypologiesEndpoint() {
+ return datasourceTypologiesEndpoint;
+ }
+
+ public void setDatasourceTypologiesEndpoint(final String datasourceTypologiesEndpoint) {
+ this.datasourceTypologiesEndpoint = datasourceTypologiesEndpoint;
+ }
+ }
+
+ public ClassPathResource getFindSolrIndexUrl() {
+ return findSolrIndexUrl;
+ }
+
+ public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) {
+ this.findSolrIndexUrl = findSolrIndexUrl;
+ }
+
+ public ClassPathResource getFindIndexDsInfo() {
+ return findIndexDsInfo;
+ }
+
+ public ClassPathResource getFindObjectStore() {
+ return findObjectStore;
+ }
+
+ public void setFindObjectStore(final ClassPathResource findObjectStore) {
+ this.findObjectStore = findObjectStore;
+ }
+
+ public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) {
+ this.findIndexDsInfo = findIndexDsInfo;
+ }
+
+ public ClassPathResource getFindFunderContexts() {
+ return findFunderContexts;
+ }
+
+ public void setFindFunderContexts(final ClassPathResource findFunderContexts) {
+ this.findFunderContexts = findFunderContexts;
+ }
+
+ public ClassPathResource getFindCommunityContexts() {
+ return findCommunityContexts;
+ }
+
+ public ClassPathResource getFindContextProfiles() {
+ return findContextProfiles;
+ }
+
+ public ClassPathResource getFindContextProfilesByType() {
+ return findContextProfilesByType;
+ }
+
+ public void setFindContextProfiles(final ClassPathResource findContextProfiles) {
+ this.findContextProfiles = findContextProfiles;
+ }
+
+ public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) {
+ this.findContextProfilesByType = findContextProfilesByType;
+ }
+
+ public void setFindCommunityContexts(final ClassPathResource findCommunityContexts) {
+ this.findCommunityContexts = findCommunityContexts;
+ }
+
+ public ClassPathResource getGetRepoProfile() {
+ return getRepoProfile;
+ }
+
+ public void setGetRepoProfile(final ClassPathResource getRepoProfile) {
+ this.getRepoProfile = getRepoProfile;
+ }
+
+ public String getContentLoadQuery() {
+ return contentLoadQuery;
+ }
+
+ public void setContentLoadQuery(String contentLoadQuery) {
+ this.contentLoadQuery = contentLoadQuery;
+ }
+
+ public String getIsLookupUrl() {
+ return isLookupUrl;
+ }
+
+ public void setIsLookupUrl(final String isLookupUrl) {
+ this.isLookupUrl = isLookupUrl;
+ }
+
+ public String getObjectStoreServiceUrl() {
+ return objectStoreServiceUrl;
+ }
+
+ public void setObjectStoreServiceUrl(final String objectStoreServiceUrl) {
+ this.objectStoreServiceUrl = objectStoreServiceUrl;
+ }
+
+ public String getIsRegistryServiceUrl() {
+ return isRegistryServiceUrl;
+ }
+
+ public void setIsRegistryServiceUrl(final String isRegistryServiceUrl) {
+ this.isRegistryServiceUrl = isRegistryServiceUrl;
+ }
+
+ public int getRequestWorkers() {
+ return requestWorkers;
+ }
+
+ public void setRequestWorkers(final int requestWorkers) {
+ this.requestWorkers = requestWorkers;
+ }
+
+ public int getRequestTimeout() {
+ return requestTimeout;
+ }
+
+ public void setRequestTimeout(final int requestTimeout) {
+ this.requestTimeout = requestTimeout;
+ }
+
+ public int getCxfClientConnectTimeout() {
+ return cxfClientConnectTimeout;
+ }
+
+ public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) {
+ this.cxfClientConnectTimeout = cxfClientConnectTimeout;
+ }
+
+ public int getCxfClientReceiveTimeout() {
+ return cxfClientReceiveTimeout;
+ }
+
+ public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) {
+ this.cxfClientReceiveTimeout = cxfClientReceiveTimeout;
+ }
+
+ public Datasource getDatasource() {
+ return datasource;
+ }
+
+ public void setDatasource(final Datasource datasource) {
+ this.datasource = datasource;
+ }
+
+ public Project getProject() {
+ return project;
+ }
+
+ public void setProject(final Project project) {
+ this.project = project;
+ }
+
+ public Jdbc getJdbc() {
+ return jdbc;
+ }
+
+ public void setJdbc(final Jdbc jdbc) {
+ this.jdbc = jdbc;
+ }
+
+ public Swagger getSwaggerDsm() {
+ return swaggerDsm;
+ }
+
+ public void setSwaggerDsm(final Swagger swaggerDsm) {
+ this.swaggerDsm = swaggerDsm;
+ }
+
+ public Swagger getSwaggerProjects() {
+ return swaggerProjects;
+ }
+
+ public void setSwaggerProjects(final Swagger swaggerProjects) {
+ this.swaggerProjects = swaggerProjects;
+ }
+
+ public Swagger getSwaggerFunders() {
+ return swaggerFunders;
+ }
+
+ public void setSwaggerFunders(final Swagger swaggerFunders) {
+ this.swaggerFunders = swaggerFunders;
+ }
+
+ public Swagger getSwaggerCommunities() {
+ return swaggerCommunities;
+ }
+
+ public void setSwaggerCommunities(final Swagger swaggerCommunities) {
+ this.swaggerCommunities = swaggerCommunities;
+ }
+
+ public Swagger getSwaggerContexts() {
+ return swaggerContexts;
+ }
+
+ public void setSwaggerContexts(final Swagger swaggerContexts) {
+ this.swaggerContexts = swaggerContexts;
+ }
+
+ public Swagger getSwaggerInfo() { return swaggerInfo; }
+
+ public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; }
+
+ public Vocabularies getVocabularies() {
+ return vocabularies;
+ }
+
+ public void setVocabularies(final Vocabularies vocabularies) {
+ this.vocabularies = vocabularies;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java
new file mode 100644
index 00000000..3276bf1c
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerConfig.java
@@ -0,0 +1,115 @@
+package eu.dnetlib;
+
+import eu.dnetlib.OpenaireExporterConfig.Swagger;
+import eu.dnetlib.openaire.community.CommunityApiController;
+import eu.dnetlib.openaire.context.ContextApiController;
+import eu.dnetlib.openaire.dsm.DsmApiController;
+import eu.dnetlib.openaire.funders.FundersApiController;
+import eu.dnetlib.openaire.info.InfoController;
+import eu.dnetlib.openaire.project.ProjectsController;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.service.ApiInfo;
+import springfox.documentation.service.Contact;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+import static springfox.documentation.builders.RequestHandlerSelectors.basePackage;
+
+@Configuration
+@EnableSwagger2
+public class SwaggerConfig {
+
+ private static final Log log = LogFactory.getLog(SwaggerConfig.class);
+
+ public static final String V1 = "1.0.0";
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ @Bean
+ public Docket dsm() {
+ return _docket(
+ "Datasource Manager",
+ DsmApiController.class.getPackage().getName(),
+ config.getSwaggerDsm(),
+ V1);
+ }
+
+ @Bean
+ public Docket projects() {
+ return _docket(
+ "OpenAIRE Projects",
+ ProjectsController.class.getPackage().getName(),
+ config.getSwaggerProjects(),
+ V1);
+ }
+
+ @Bean
+ public Docket funders() {
+ return _docket(
+ "OpenAIRE Funders",
+ FundersApiController.class.getPackage().getName(),
+ config.getSwaggerFunders(),
+ V1);
+ }
+
+ @Bean
+ public Docket communities() {
+ return _docket(
+ "OpenAIRE Communities",
+ CommunityApiController.class.getPackage().getName(),
+ config.getSwaggerCommunities(),
+ V1);
+ }
+
+ @Bean
+ public Docket contexts() {
+ return _docket(
+ "OpenAIRE Contexts",
+ ContextApiController.class.getPackage().getName(),
+ config.getSwaggerCommunities(),
+ V1);
+ }
+
+ @Bean
+ public Docket info() {
+ return _docket(
+ "OpenAIRE Info",
+ InfoController.class.getPackage().getName(),
+ config.getSwaggerInfo(),
+ V1);
+ }
+
+ private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) {
+ return new Docket(DocumentationType.SWAGGER_2)
+ .groupName(groupName)
+ .select()
+ .apis(basePackage(controllerPackage))
+ .build()
+ .directModelSubstitute(org.joda.time.LocalDate.class, java.sql.Date.class)
+ .directModelSubstitute(org.joda.time.DateTime.class, java.util.Date.class)
+ .apiInfo(apiInfo(swag, version));
+ }
+
+ private ApiInfo apiInfo(final Swagger swag, final String version) {
+ return new ApiInfoBuilder()
+ .title(swag.getApiTitle())
+ .description(swag.getApiDescription())
+ .license(swag.getApiLicense())
+ .licenseUrl(swag.getApiLicenseUrl())
+ .termsOfServiceUrl("")
+ .version(version)
+ .contact(new Contact(
+ swag.getApiContactName(),
+ swag.getApiContactUrl(),
+ swag.getApiContactEmail()))
+ .build();
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java
new file mode 100644
index 00000000..c11e15f0
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java
@@ -0,0 +1,100 @@
+package eu.dnetlib.openaire.common;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.MethodArgumentNotValidException;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+/**
+ * Created by claudio on 18/07/2017.
+ */
+public abstract class AbstractExporterController {
+
+ private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @ResponseBody
+ @ExceptionHandler({DsmException.class})
+ @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
+ public ErrorMessage handleDSMException(final Exception e) {
+ return _handleError(e);
+ }
+
+ @ResponseBody
+ @ExceptionHandler(DsmForbiddenException.class)
+ @ResponseStatus(value = HttpStatus.FORBIDDEN)
+ public ErrorMessage handleForbiddenException(final Exception e) {
+ return _handleError(e);
+ }
+
+ @ResponseBody
+ @ExceptionHandler({DsmNotFoundException.class})
+ @ResponseStatus(value = HttpStatus.NOT_FOUND)
+ public ErrorMessage handleNotFoundException(final Exception e) {
+ return _handleError(e);
+ }
+
+ @ResponseBody
+ @ExceptionHandler(MethodArgumentNotValidException.class)
+ @ResponseStatus(HttpStatus.BAD_REQUEST)
+ public List processValidationError(final MethodArgumentNotValidException e) {
+ return e.getBindingResult()
+ .getFieldErrors().stream()
+ .map(fe -> new ErrorMessage(
+ String.format("field '%s'", fe.getField()),
+ String.format("rejected value '%s'", fe.getRejectedValue()),
+ fe.getDefaultMessage()))
+ .collect(Collectors.toList());
+ }
+
+ private ErrorMessage _handleError(final Exception e) {
+ log.error(e);
+ if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
+ return null; //socket is closed, cannot return any response
+ } else {
+ return new ErrorMessage(e);
+ }
+ }
+
+ @JsonAutoDetect
+ public class ErrorMessage {
+
+ private final String message;
+ private final String details;
+ private final String stacktrace;
+
+ public ErrorMessage(final Exception e) {
+ this(e.getMessage(), "", ExceptionUtils.getStackTrace(e));
+ }
+
+ public ErrorMessage(final String message, final String details, final String stacktrace) {
+ this.message = message;
+ this.details = details;
+ this.stacktrace = stacktrace;
+ }
+
+ public String getMessage() {
+ return this.message;
+ }
+
+ public String getStacktrace() {
+ return this.stacktrace;
+ }
+
+ public String getDetails() {
+ return details;
+ }
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java
new file mode 100644
index 00000000..9f7d04e3
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java
@@ -0,0 +1,58 @@
+package eu.dnetlib.openaire.common;
+
+import java.sql.Array;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+import javax.sql.DataSource;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
+/**
+ * Created by claudio on 05/07/2017.
+ */
+@Converter
+public class ConverterTextArray implements AttributeConverter, Array>, ApplicationContextAware {
+
+ private ApplicationContext applicationContext;
+
+ @Override
+ public Array convertToDatabaseColumn(List attribute) {
+
+ final Map datasources = applicationContext.getBeansOfType(DataSource.class);
+ DataSource source = datasources.values().stream().findFirst().get();
+
+ try {
+ Connection conn = source.getConnection();
+ return conn.createArrayOf("text", attribute.toArray());
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+
+ }
+
+ @Override
+ public List convertToEntityAttribute(Array dbData) {
+ try {
+ return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList());
+ } catch (SQLException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ @Override
+ public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
+ this.applicationContext = applicationContext;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java
new file mode 100644
index 00000000..5fd8b6f7
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java
@@ -0,0 +1,47 @@
+package eu.dnetlib.openaire.common;
+
+public class ExporterConstants {
+
+ /*
+ Tags used to group the operations on the swagger UI
+ */
+ public final static String C = "Community";
+ public final static String C_CP = "Community content providers";
+ public final static String C_PJ = "Community projects";
+ public final static String C_ZC = "Community Zenodo Communities";
+ public final static String C_O = "Community Organizations";
+
+ public final static String DS = "Datasource";
+ public final static String API = "Interface";
+ public final static String R = "Read";
+ public final static String W = "Write";
+
+ public final static String D = "Deprecated";
+ public final static String M = "Management";
+
+ public final static String DSPACE = "DSpace";
+ public final static String EPRINT = "EPrints";
+ public final static String TSV = "TSV";
+ public final static String STREAMING = "Streaming";
+
+ public static final String OAI = "oai";
+ public static final String SET = "set";
+
+ // Paths used in the repository profile mapping: Datasource
+ public static final String ADMIN_INFO = "//CONFIGURATION/ADMIN_INFO";
+ public static final String ENGLISH_NAME = "//CONFIGURATION/ENGLISH_NAME";
+ public static final String OFFICIAL_NAME = "//CONFIGURATION/OFFICIAL_NAME";
+ public static final String TIMEZONE = "//CONFIGURATION/LOCATION/TIMEZONE";
+ public static final String LATITUDE = "//CONFIGURATION/LOCATION/LATITUDE";
+ public static final String LONGITUDE = "//CONFIGURATION/LOCATION/LONGITUDE";
+ public static final String PLATFORM = "//CONFIGURATION/TYPOLOGY";
+ public static final String TYPOLOGY = "//CONFIGURATION/DATASOURCE_TYPE";
+ public static final String WEBSITE_URL = "//CONFIGURATION/REPOSITORY_WEBPAGE";
+
+ // Paths used in the repository profile mapping: Interface
+ public static final String OAI_SET = "/ACCESS_PROTOCOL/@set";
+ public static final String BASE_URL = "/BASE_URL";
+ public static final String COMPLIANCE = "/@compliance";
+ public static final String REMOVABLE = "/@removable";
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java
new file mode 100644
index 00000000..51a4d738
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java
@@ -0,0 +1,106 @@
+package eu.dnetlib.openaire.common;
+
+import java.io.Serializable;
+import java.sql.*;
+
+import org.hibernate.HibernateException;
+import org.hibernate.engine.spi.SharedSessionContractImplementor;
+import org.hibernate.usertype.UserType;
+
+/**
+ * Created by claudio on 05/07/2017.
+ */
+public class GenericArrayUserType implements UserType {
+
+ protected static final int[] SQL_TYPES = { Types.ARRAY };
+
+ private Class typeParameterClass;
+
+ @Override
+ public Object assemble(Serializable cached, Object owner) throws HibernateException {
+ return this.deepCopy(cached);
+ }
+
+ @Override
+ public Object deepCopy(Object value) throws HibernateException {
+ return value;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Serializable disassemble(Object value) throws HibernateException {
+ return (T) this.deepCopy(value);
+ }
+
+ @Override
+ public boolean equals(Object x, Object y) throws HibernateException {
+
+ if (x == null) {
+ return y == null;
+ }
+ return x.equals(y);
+ }
+
+ @Override
+ public int hashCode(Object x) throws HibernateException {
+ return x.hashCode();
+ }
+
+ @Override
+ public Object nullSafeGet(final ResultSet resultSet,
+ final String[] names,
+ final SharedSessionContractImplementor sharedSessionContractImplementor,
+ final Object o)
+ throws HibernateException, SQLException {
+ if (resultSet.wasNull()) {
+ return null;
+ }
+ if (resultSet.getArray(names[0]) == null) {
+ return new Integer[0];
+ }
+
+ Array array = resultSet.getArray(names[0]);
+ @SuppressWarnings("unchecked")
+ T javaArray = (T) array.getArray();
+ return javaArray;
+ }
+
+ @Override
+ public void nullSafeSet(final PreparedStatement statement,
+ final Object value,
+ final int index,
+ final SharedSessionContractImplementor session)
+ throws HibernateException, SQLException {
+ Connection connection = statement.getConnection();
+ if (value == null) {
+ statement.setNull(index, SQL_TYPES[0]);
+ } else {
+ @SuppressWarnings("unchecked")
+ T castObject = (T) value;
+ Array array = connection.createArrayOf("integer", (Object[]) castObject);
+ statement.setArray(index, array);
+ }
+ }
+
+ @Override
+ public boolean isMutable() {
+ return true;
+ }
+
+ @Override
+ public Object replace(Object original, Object target, Object owner) throws HibernateException {
+ return original;
+ }
+
+ @Override
+ public Class returnedClass() {
+ return typeParameterClass;
+ }
+
+ @Override
+ public int[] sqlTypes() {
+ return new int[] { Types.ARRAY };
+ }
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java
new file mode 100644
index 00000000..848654cc
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java
@@ -0,0 +1,60 @@
+package eu.dnetlib.openaire.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.openaire.context.Context;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
+import eu.dnetlib.openaire.dsm.domain.ApiDetails;
+import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
+
+public interface ISClient {
+
+ IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException;
+
+ String getObjectStoreId(String dsId) throws DsmException;
+
+ Map getFunderContextMap() throws IOException;
+
+ Map getCommunityContextMap() throws IOException;
+
+ Map getContextMap(final List type) throws IOException;
+
+ void updateContextParam(String id, String name, String value);
+
+ void updateContextAttribute(String id, String name, String value);
+
+ void addConcept(String id, String categoryId, String data);
+
+ void removeConcept(String id, String categoryId, String conceptId);
+
+ void updateDatasourceFields(String dsId, Map changes);
+
+ void addAPIAttribute(String dsId, String apiId, Map changes);
+
+ void updateAPIField(String dsId, String apiId, Map changes);
+
+ void registerDS(DatasourceDetails d);
+
+ void registerAPI(ApiDetails api);
+
+ void removeAPI(String apiId) throws DsmForbiddenException;
+
+ void dropCache();
+
+ /**
+ *
+ * @param id id of the concept to be updated (i.e. ni::projects::2)
+ * @param name name of the attribute to be updated
+ * @param value new value for the attribute
+ */
+ void updateConceptAttribute(String id, String name, String value);
+
+ void updateConceptParam(String id, String name, String value);
+
+ void updateConceptParamNoEscape(String id, String name, String value);
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java
new file mode 100644
index 00000000..07e28df9
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java
@@ -0,0 +1,413 @@
+package eu.dnetlib.openaire.common;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import com.google.common.escape.Escaper;
+import com.google.common.xml.XmlEscapers;
+import eu.dnetlib.OpenaireExporterConfig;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.enabling.datasources.common.DsmRuntimeException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
+import eu.dnetlib.openaire.context.Context;
+import eu.dnetlib.openaire.context.ContextMappingUtils;
+import eu.dnetlib.openaire.dsm.domain.ApiDetails;
+import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryInterfce;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asRepositoryProfile;
+import static eu.dnetlib.openaire.common.Utils.escape;
+
+/**
+ * Created by claudio on 20/10/2016.
+ */
+@Component
+public class ISClientImpl implements ISClient {
+
+ private static final Log log = LogFactory.getLog(ISClientImpl.class);
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ @Autowired
+ private ISLookUpService isLookUpService;
+
+ @Autowired
+ private ISRegistryService isRegistryService;
+
+ @Autowired
+ private OperationManager operationManager;
+
+ @Override
+ @Cacheable("indexdsinfo-cache")
+ public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException {
+ log.warn("calculateCurrentIndexDsInfo(): not using cache");
+ final String[] arr;
+ try {
+ arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@");
+ return new IndexDsInfo(
+ _isLookUp(_getQuery(config.getFindSolrIndexUrl())),
+ arr[0].trim(), arr[1].trim(), arr[2].trim());
+ } catch (IOException | ISLookUpException e) {
+ throw new DsmException("unable fetch index DS information from IS");
+ }
+ }
+
+ @Override
+ @Cacheable("objectstoreid-cache")
+ public String getObjectStoreId(final String dsId) throws DsmException {
+ log.warn(String.format("getObjectStoreId(%s): not using cache", dsId));
+ try {
+ final String xqueryTemplate = _getQuery(config.getFindObjectStore());
+ return _isLookUp(String.format(xqueryTemplate, dsId));
+ } catch (IOException | ISLookUpException e) {
+ throw new DsmException("unble to find objectstore for ds " + dsId);
+ }
+ }
+
+ @Override
+ @Cacheable("context-cache-funder")
+ public Map getFunderContextMap() throws IOException {
+ return _processContext(_getQuery(config.getFindFunderContexts()));
+ }
+
+ @Override
+ @Cacheable("context-cache-community")
+ public Map getCommunityContextMap() throws IOException {
+ return _processContext(_getQuery(config.getFindCommunityContexts()));
+ }
+
+ @Override
+ @Cacheable("context-cache")
+ public Map getContextMap(final List type) throws IOException {
+ if (Objects.isNull(type) || type.isEmpty()) {
+ return _processContext(_getQuery(config.getFindContextProfiles()));
+ } else {
+ final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
+
+ final String xquery = String.format(xqueryTemplate, type.stream()
+ .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
+ .collect(Collectors.joining(" or ")));
+
+ return _processContext(xquery);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void updateContextParam(final String id, final String name, final String value) {
+ try {
+ _quickSeachProfile(getXQuery(id, name, value));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void updateContextAttribute(final String id, final String name, final String value) {
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ try {
+ _quickSeachProfile(String.format(
+ "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void addConcept(final String id, final String categoryId, final String data) {
+ try {
+ _quickSeachProfile(String.format(
+ "update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void removeConcept(final String id, final String categoryId, final String conceptId) {
+ try {
+ _quickSeachProfile(String.format(
+ "for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" +
+ "/category[./@id = '%s']/concept[./@id = '%s'] " +
+ "return update delete $concept", id, categoryId, conceptId));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-community", "context-cache-funder"}, allEntries = true)
+ public void updateConceptAttribute(String id, String name, String value) {
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ try {
+ _quickSeachProfile(String.format(
+ "update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e);
+ }
+
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void updateConceptParam(String id, String name, String value) {
+ try {
+ _quickSeachProfile(getConceptXQuery(id, name, value));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
+ }
+ }
+
+ @Override
+ @CacheEvict(value = { "context-cache", "context-cache-funder"}, allEntries = true)
+ public void updateConceptParamNoEscape(String id, String name, String value) {
+ try {
+ _quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
+ }
+ }
+
+ @Override
+ public void updateDatasourceFields(final String dsId, final Map changes) {
+ operationManager.addOperation(() -> {
+ Thread.currentThread().setName("update-ds:" + dsId);
+ changes.forEach((xpath, value) -> {
+ try {
+ _isLookUp(String.format(
+ "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" +
+ "where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE']/text() = '%s' \n" +
+ "return update value $x%s with '%s'", dsId, xpath, value));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update datasource fields [id: %s, changes: %s]", dsId, changes), e);
+ }
+ });
+ });
+ }
+
+ @Override
+ public void addAPIAttribute(final String dsId, final String apiId, final Map changes) {
+ operationManager.addOperation(() -> {
+ Thread.currentThread().setName("update-api:" + dsId);
+ changes.forEach((xpath, value) -> {
+ try {
+ final String attribute = StringUtils.substringAfter(xpath, "@");
+ final String parentElement = StringUtils.substringBeforeLast(xpath, "/");
+ _isLookUp(String.format(
+ "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" +
+ "return update insert attribute %s {'%s'} into $x/..//INTERFACE[./@id = '%s']%s",
+ dsId, attribute, value, apiId, parentElement));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable add API attribute [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e);
+ }
+ });
+ });
+ }
+
+ @Override
+ public void updateAPIField(final String dsId, final String apiId, final Map changes) {
+ operationManager.addOperation(() -> {
+ Thread.currentThread().setName("update-api:" + dsId);
+ changes.forEach((xpath, value) -> {
+ try {
+ _isLookUp(String.format(
+ "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" +
+ "return update value $x/..//INTERFACE[./@id = '%s']%s with '%s'",
+ dsId, apiId, xpath, value));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable update API fields [dsId: %s, apiId: %s, changes: %s]", dsId, apiId, changes), e);
+ }
+ });
+ });
+ }
+
+ @Override
+ public void registerDS(final DatasourceDetails d) {
+ operationManager.addOperation(() -> {
+ Thread.currentThread().setName("save-ds:" + d.getId());
+ try {
+ final String id = isRegistryService.registerProfile(asRepositoryProfile(d));
+ log.debug(String.format("registered DS profile %s", id));
+ } catch (ISRegistryException e) {
+ throw new DsmRuntimeException("unable to register DS profile: " + d.getId(), e);
+ }
+ });
+ }
+
+ @Override
+ public void registerAPI(final ApiDetails api) {
+ operationManager.addOperation(() -> {
+ Thread.currentThread().setName("save-api:" + api.getId());
+ try {
+ final String dsId = api.getDatasource();
+ final String iface = asRepositoryInterfce(api);
+ _isLookUp(String.format(
+ "let $x:=/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance='OPENAIRE' and ./text() = '%s']\n" +
+ "return update insert %s into $x/../INTERFACES", dsId, iface));
+
+ log.debug(String.format("registered API %s", api.getId()));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException("unable to register API: " + api.getId(), e);
+ }
+ });
+ }
+
+ @Override
+ public void removeAPI(final String apiId) throws DsmForbiddenException {
+ try {
+ final List metaWorkflows = _quickSeachProfile(String.format(
+ "distinct-values(for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')\n" +
+ "where $x/RESOURCE_PROFILE/BODY/DATAPROVIDER[./@interface = '%s']\n" +
+ "return $x/RESOURCE_PROFILE/BODY/DATAPROVIDER/@id/string())", apiId));
+ if (!metaWorkflows.isEmpty()) {
+ throw new DsmForbiddenException(
+ HttpStatus.SC_FORBIDDEN,
+ String.format("cannot remove api '%s', it has workflows associated", apiId));
+ }
+ isLookUpService.quickSearchProfile(String.format(
+ " update delete /RESOURCE_PROFILE/BODY/CONFIGURATION/INTERFACES/INTERFACE[./@id = '%s']", apiId));
+
+ log.info(String.format("deleted API %s", apiId));
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException(String.format("unable to remove API %s", apiId), e);
+ }
+ }
+
+ /// HELPERS
+
+ private String getXQuery(final String id, final String name, final String value) {
+ final Escaper esc = XmlEscapers.xmlContentEscaper();
+ if (StringUtils.isNotBlank(value)) {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name,
+ escape(esc, value));
+ } else {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
+ "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name);
+ }
+ }
+
+ private String getConceptXQuery(final String id, final String name, final String value) {
+ final Escaper esc = XmlEscapers.xmlContentEscaper();
+ if (StringUtils.isNotBlank(value)) {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
+ "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name,
+ escape(esc, value));
+ } else {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name);
+ }
+ }
+
+ private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
+
+ if (StringUtils.isNotBlank(value)) {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
+ "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name,
+ value);
+ } else {
+ return String.format(
+ "update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name);
+ }
+ }
+
+ private Map _processContext(final String xquery) throws IOException {
+ return _processContext(new LinkedBlockingQueue<>(), xquery);
+ }
+
+ private Map _processContext(final Queue errors, final String xquery) throws IOException {
+ try {
+ return getContextProfiles(errors, xquery).stream()
+ .filter(StringUtils::isNotBlank)
+ .map(s -> ContextMappingUtils.parseContext(s, errors))
+ .collect(Collectors.toMap(
+ Context::getId,
+ Function.identity(),
+ (c1, c2) -> {
+ log.warn(String.format("found duplicate context profile '%s'", c1.getId()));
+ return c1;
+ }));
+ } finally {
+ if (!errors.isEmpty()) {
+ log.error(errors);
+ errors.forEach(Throwable::printStackTrace);
+ }
+ }
+ }
+
+ private List getContextProfiles(final Queue errors, final String xquery) throws IOException {
+ log.warn("getContextProfiles(): not using cache");
+ try {
+ return _quickSeachProfile(xquery);
+ } catch (ISLookUpException e) {
+ throw new DsmRuntimeException("unable to get context profiles", e);
+ }
+ }
+
+ private String _getQuery(final ClassPathResource resource) throws IOException {
+ return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset());
+ }
+
+ private String _isLookUp(final String xquery) throws ISLookUpException {
+ log.debug(String.format("running xquery:\n%s", xquery));
+ //log.debug(String.format("query result: %s", res));
+ return isLookUpService.getResourceProfileByQuery(xquery);
+ }
+
+ private List _quickSeachProfile(final String xquery) throws ISLookUpException {
+ final List res = Lists.newArrayList();
+
+ log.debug(String.format("running xquery:\n%s", xquery));
+ final List list = isLookUpService.quickSearchProfile(xquery);
+ if (list != null) {
+ res.addAll(list);
+ }
+ log.debug(String.format("query result size: %s", res.size()));
+ return res;
+ }
+
+ @CacheEvict(cacheNames = { "context-cache", "indexdsinfo-cache", "objectstoreid-cache" }, allEntries = true)
+ @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
+ public void dropCache() {
+ log.debug("dropped dsManager IS cache");
+ }
+
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java
new file mode 100644
index 00000000..532c1890
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java
@@ -0,0 +1,60 @@
+package eu.dnetlib.openaire.common;
+
+import java.util.List;
+import java.util.concurrent.*;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class OperationManager {
+
+ private static final Log log = LogFactory.getLog(OperationManager.class);
+
+ private static final long SLEEP_TIME = 1000;
+
+ private static final int Q_SIZE = 100;
+
+ private static final int POOL_SIZE = 5;
+
+ private final BlockingQueue ops = new ArrayBlockingQueue<>(Q_SIZE);
+
+ private ExecutorService executor;
+
+ @PostConstruct
+ public void init() {
+ executor = getExecutor();
+ }
+
+ public int dropAll() {
+ final List lostOperations = executor.shutdownNow();
+ log.warn(String.format("discarding %s operations", lostOperations.size()));
+ executor = getExecutor();
+ return lostOperations.size();
+ }
+
+ public int getOpSize() {
+ return ops.size();
+ }
+
+ public void addOperation(final Runnable op) {
+ executor.execute(op);
+ }
+
+ @PreDestroy
+ public void tearDown() throws InterruptedException {
+ executor.shutdown();
+ final boolean done = executor.awaitTermination(SLEEP_TIME, TimeUnit.MILLISECONDS);
+ log.debug(String.format("All operations were completed so far? %s", done));
+ }
+
+ // HELPERS
+
+ private ThreadPoolExecutor getExecutor() {
+ return new ThreadPoolExecutor(POOL_SIZE, POOL_SIZE,0L, TimeUnit.MILLISECONDS, ops);
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java
new file mode 100644
index 00000000..28fe4e3f
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java
@@ -0,0 +1,60 @@
+package eu.dnetlib.openaire.common;
+
+import java.text.FieldPosition;
+import java.util.*;
+
+import com.fasterxml.jackson.databind.util.StdDateFormat;
+
+public class RFC3339DateFormat extends StdDateFormat {
+
+ private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC");
+
+ // Same as ISO8601DateFormat but serializing milliseconds.
+ @Override
+ public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition fieldPosition) {
+ String value = format(date, true, TIMEZONE_Z, Locale.US);
+ toAppendTo.append(value);
+ return toAppendTo;
+ }
+
+ /**
+ * Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
+ *
+ * @param date the date to format
+ * @param millis true to include millis precision otherwise false
+ * @param tz timezone to use for the formatting (UTC will produce 'Z')
+ * @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
+ */
+ private static String format(Date date, boolean millis, TimeZone tz, Locale loc) {
+ Calendar calendar = new GregorianCalendar(tz, loc);
+ calendar.setTime(date);
+
+ // estimate capacity of buffer as close as we can (yeah, that's pedantic ;)
+ StringBuilder sb = new StringBuilder(30);
+ sb.append(String.format(
+ "%04d-%02d-%02dT%02d:%02d:%02d",
+ calendar.get(Calendar.YEAR),
+ calendar.get(Calendar.MONTH) + 1,
+ calendar.get(Calendar.DAY_OF_MONTH),
+ calendar.get(Calendar.HOUR_OF_DAY),
+ calendar.get(Calendar.MINUTE),
+ calendar.get(Calendar.SECOND)
+ ));
+ if (millis) {
+ sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND)));
+ }
+
+ int offset = tz.getOffset(calendar.getTimeInMillis());
+ if (offset != 0) {
+ int hours = Math.abs((offset / (60 * 1000)) / 60);
+ int minutes = Math.abs((offset / (60 * 1000)) % 60);
+ sb.append(String.format("%c%02d:%02d",
+ (offset < 0 ? '-' : '+'),
+ hours, minutes));
+ } else {
+ sb.append('Z');
+ }
+ return sb.toString();
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java
new file mode 100644
index 00000000..600e29d6
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java
@@ -0,0 +1,22 @@
+package eu.dnetlib.openaire.common;
+
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import com.google.common.escape.Escaper;
+import org.apache.commons.lang3.StringUtils;
+
+public class Utils {
+
+ public static Stream stream(Iterator iterator) {
+ return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
+ }
+
+ public static String escape(final Escaper esc, final String value) {
+ return StringUtils.isNotBlank(value) ? esc.escape(value) : "";
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java
new file mode 100644
index 00000000..e653b09a
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java
@@ -0,0 +1,304 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.List;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.web.bind.annotation.*;
+
+import static eu.dnetlib.openaire.common.ExporterConstants.*;
+
+@RestController
+@CrossOrigin(origins = { "*" })
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+@io.swagger.annotations.Api(tags = "OpenAIRE Communities API", description = "the OpenAIRE Community API")
+public class CommunityApiController {
+
+ @Autowired
+ private CommunityApiCore communityApiCore;
+
+
+ @RequestMapping(value = "/community/communities", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get all community profiles",
+ notes = "get all community profiles",
+ tags = { C, R },
+ response = CommunitySummary[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunitySummary[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public List listCommunities() throws CommunityException {
+ return communityApiCore.listCommunities();
+ }
+
+ @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get community profile",
+ notes = "get community profile",
+ tags = { C, R },
+ response = CommunityDetails.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunityDetails.class),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityDetails getCommunity(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
+ return communityApiCore.getCommunity(id);
+ }
+
+ @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "update community details",
+ notes = "update community details",
+ tags = { C, R })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public void setCommunity(
+ @PathVariable final String id,
+ @RequestBody CommunityWritableProperties properties) throws CommunityException, CommunityNotFoundException {
+
+ communityApiCore.setCommunity(id, properties);
+ }
+
+ @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get community projects",
+ notes = "get community projects",
+ tags = { C_PJ, R },
+ response = CommunityProject[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunityProject[].class),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public List getCommunityProjects(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
+ return communityApiCore.getCommunityProjects(id);
+ }
+
+ @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "associate a project to the community",
+ notes = "associate a project to the community",
+ tags = { C_PJ, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityProject addCommunityProject(
+ @PathVariable final String id,
+ @RequestBody final CommunityProject project) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.addCommunityProject(id, project);
+ }
+
+ @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.DELETE)
+ @ApiOperation(
+ value = "remove a project from the community",
+ notes = "remove a project from the community",
+ tags = { C_PJ, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public void deleteCommunityProject(
+ @PathVariable final String id,
+ @RequestBody final Integer projectId) throws CommunityException, CommunityNotFoundException {
+
+ communityApiCore.removeCommunityProject(id, projectId);
+ }
+
+ @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get the list of content providers associated to a given community",
+ notes = "get the list of content providers associated to a given community",
+ tags = { C_CP, R },
+ response = CommunityContentprovider[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public List getCommunityContentproviders(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
+ return communityApiCore.getCommunityContentproviders(id);
+ }
+
+ @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "associate a content provider to the community",
+ notes = "associate a content provider to the community",
+ tags = { C_CP, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityContentprovider addCommunityContentprovider(
+ @PathVariable final String id,
+ @RequestBody final CommunityContentprovider contentprovider) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.addCommunityContentprovider(id, contentprovider);
+ }
+
+ @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.DELETE)
+ @ApiOperation(
+ value = "remove the association between a content provider and the community",
+ notes = "remove the association between a content provider and the community",
+ tags = { C_CP, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public void removeCommunityContentprovider(
+ @PathVariable final String id,
+ @RequestBody final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
+
+ communityApiCore.removeCommunityContentProvider(id, contentproviderId);
+ }
+
+ //ADDING CODE FOR COMMUNITY ORGANIZATIONS
+
+ @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get the list of organizations for a given community",
+ notes = "get the list of organizations for a given community",
+ tags = { C_O, R },
+ response = CommunityOrganization[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public List getCommunityOrganizations(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
+ return communityApiCore.getCommunityOrganizations(id);
+ }
+
+ @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "associate an organization to the community",
+ notes = "associate an organization to the community",
+ tags = { C_O, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityOrganization addCommunityOrganization(
+ @PathVariable final String id,
+ @RequestBody final CommunityOrganization organization) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.addCommunityOrganization(id, organization);
+ }
+
+ @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.DELETE)
+ @ApiOperation(
+ value = "remove the association between an organization and the community",
+ notes = "remove the association between an organization and the community",
+ tags = { C_O, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public void removeCommunityOrganization(
+ @PathVariable final String id,
+ @RequestBody final Integer organizationId) throws CommunityException, CommunityNotFoundException {
+
+ communityApiCore.removeCommunityOrganization(id, organizationId);
+ }
+ //**********************
+
+ @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "associate a subject to the community",
+ notes = "associate a subject to the community",
+ tags = { C, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityDetails addCommunitySubjects(
+ @PathVariable final String id,
+ @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.addCommunitySubjects(id, subjects);
+ }
+
+ @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.DELETE)
+ @ApiOperation(
+ value = "remove subjects from a community",
+ notes = "remove subjects from a community",
+ tags = { C, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityDetails removeCommunitySubjects(
+ @PathVariable final String id,
+ @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.removeCommunitySubjects(id, subjects);
+ }
+
+ @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get the list of Zenodo communities associated to a given community",
+ notes = "get the list of Zenodo communities associated to a given community",
+ tags = { C_ZC, R },
+ response = CommunityZenodoCommunity[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CommunityZenodoCommunity[].class),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public List getCommunityZenodoCommunities(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
+ return communityApiCore.getCommunityZenodoCommunities(id);
+ }
+
+ @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(
+ value = "associate a Zenodo community to the community",
+ notes = "associate a Zenodo community to the community",
+ tags = { C_ZC, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityZenodoCommunity addCommunityZenodoCommunity(
+ @PathVariable final String id,
+ @RequestBody final CommunityZenodoCommunity zenodocommunity) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.addCommunityZenodoCommunity(id, zenodocommunity);
+
+ }
+
+ @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.DELETE)
+ @ApiOperation(
+ value = "remove a Zenodo community from a community",
+ notes = "remove a Zenodo community from a community",
+ tags = { C_ZC, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public void removeCommunityZenodoCommunity(
+ @PathVariable final String id,
+ @RequestBody final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
+
+ communityApiCore.removeCommunityZenodoCommunity(id, zenodoCommId);
+
+ }
+
+
+ @RequestMapping(value = "/community/{zenodoId}/openairecommunities", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "get the list of OpenAIRE communities associated to a given Zenodo community",
+ notes = "get the list of OpenAIRE communities associated to a given Zenodo community",
+ tags = { C_ZC, R })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
+ public CommunityOpenAIRECommunities getOpenAireCommunities(
+ @PathVariable final String zenodoId) throws CommunityException, CommunityNotFoundException {
+
+ return communityApiCore.getOpenAIRECommunities(zenodoId);
+
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java
new file mode 100644
index 00000000..c655a91a
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java
@@ -0,0 +1,386 @@
+package eu.dnetlib.openaire.community;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import eu.dnetlib.openaire.common.ISClient;
+import eu.dnetlib.openaire.context.Context;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static eu.dnetlib.openaire.community.CommunityConstants.*;
+
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
+public class CommunityApiCore {//implements CommunityClient{
+
+ private static final Log log = LogFactory.getLog(CommunityApiCore.class);
+
+ @Autowired
+ private CommunityClient cci;
+
+ @Autowired
+ private ISClient isClient;
+
+ @Autowired
+ private CommunityCommon cc;
+
+
+ public List listCommunities() throws CommunityException {
+ return cc.listCommunities();
+
+ }
+
+ public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
+ return cc.getCommunity(id);
+
+ }
+
+ public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException {
+
+ cc.getCommunity(id); // ensure the community exists.
+
+ if(details.getShortName() != null) {
+ isClient.updateContextAttribute(id, CLABEL, details.getShortName());
+
+ }
+ if (details.getName() != null){
+ isClient.updateContextParam(id, CSUMMARY_NAME, details.getName());
+
+ }
+ if(details.getDescription() != null) {
+ isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription());
+
+ }
+ if(details.getLogoUrl()!=null){
+ isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl());
+
+ }
+ if (details.getStatus() != null) {
+ isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name());
+
+ }
+ if (details.getSubjects() != null) {
+ isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()));
+
+ }
+ if (details.getMainZenodoCommunity() != null) {
+ isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity());
+ }
+
+ cc.updateCommunity(id, details);
+ }
+
+ public List getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException {
+ cc.getCommunity(id); // ensure the community exists.
+ return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
+ }
+
+ public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException {
+ if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) {
+ throw new CommunityException("parameters 'id' and project.communityId must be coherent");
+ }
+
+ final TreeMap projects = getCommunityProjectMap(id);
+ String project_id = project.getId();
+
+ if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))){
+ if (project.getName() != null) {
+ isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME,project.getName());
+
+ }
+ if(project.getAcronym()!= null){
+ isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM,project.getAcronym());
+
+ }
+ if (project.getOpenaireId() != null){
+ isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId());
+
+ }
+ if (project.getFunder() != null){
+ isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder());
+
+ }
+ if(project.getGrantId() != null){
+ isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId());
+
+ }
+ }else {
+ project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0));
+
+ isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project));
+
+ }
+ cc.updateProject(id, project );
+ return project;
+ }
+
+ private String nextId(final Integer id) {
+ return String.valueOf(id + 1);
+ }
+
+ public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException {
+ final Map projects = getCommunityProjectMap(id);
+ if (!projects.containsKey(projectId)) {
+ throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id));
+ }
+ isClient.removeConcept(
+ id,
+ id + PROJECTS_ID_SUFFIX,
+ id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId);
+ cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId));
+ }
+
+ public List getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException {
+ cc.getCommunity(id); // ensure the community exists.
+ return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
+ }
+
+ public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp) throws CommunityException, CommunityNotFoundException {
+ log.info("content provider to add " + cp.toString());
+ if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) {
+ throw new CommunityException("parameters 'id' and cp.communityId must be coherent");
+ }
+
+ final TreeMap cps = getCommunityContentproviderMap(id);
+ final String concept_id = cp.getId();
+ if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))){
+ if (cp.getName() != null) {
+ isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME,cp.getName());
+ }
+ if(cp.getOfficialname()!= null){
+ isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME,cp.getOfficialname());
+ }
+ if (cp.getOpenaireId() != null){
+ isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID,cp.getOpenaireId());
+ }
+ if(cp.getSelectioncriteria() != null){
+ isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML());
+ }
+ }else{
+ log.info("adding new concept for community " + id);
+ cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
+
+
+ isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp));
+ }
+
+ cc.updateDatasource(id, cp);
+ return cp;
+ }
+
+ public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
+ final Map providers = getCommunityContentproviderMap(id);
+ if (!providers.containsKey(contentproviderId)) {
+ throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id));
+ }
+ isClient.removeConcept(
+ id,
+ id + CONTENTPROVIDERS_ID_SUFFIX,
+ id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId);
+ cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId));
+ }
+
+ public void removeCommunityOrganization(String id, Integer organizationId) throws CommunityException, CommunityNotFoundException {
+ final Map organizations = getCommunityOrganizationMap(id);
+ if (!organizations.containsKey(organizationId)) {
+ throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id));
+ }
+ isClient.removeConcept(
+ id,
+ id + ORGANIZATION_ID_SUFFIX,
+ id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId);
+ cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId));
+ }
+
+ public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
+
+ return cc.getCommunityZenodoCommunities(id);
+ }
+
+ public List getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException {
+ cc.getCommunity(id);
+ return cc.getCommunityInfo(id,ORGANIZATION_ID_SUFFIX,c->CommunityMappingUtils.asCommunityOrganization(id,c));
+ }
+
+
+ public CommunityDetails addCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException {
+
+ final CommunityDetails cd = new CommunityDetails();
+
+ final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
+
+ current.addAll(subjects);
+
+ cd.setSubjects(Lists.newArrayList(current));
+
+ setCommunity(id, CommunityWritableProperties.fromDetails(cd));
+
+ return cd;
+ }
+
+ public CommunityDetails removeCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException {
+
+ final CommunityDetails cd = new CommunityDetails();
+
+ final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
+
+ current.removeAll(subjects);
+
+ cd.setSubjects(Lists.newArrayList(current));
+
+ setCommunity(id, CommunityWritableProperties.fromDetails(cd));
+
+ return cd;
+ }
+
+ @CacheEvict(value="community-cache",allEntries = true)
+ public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
+
+ final Map zcomms = getZenodoCommunityMap(id);
+ if (!zcomms.containsKey(zenodoCommId)) {
+ throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
+ }
+ isClient.removeConcept(
+ id,
+ id + ZENODOCOMMUNITY_ID_SUFFIX,
+ id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
+
+
+ cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
+
+ }
+
+ @CacheEvict(value="community-cache",allEntries = true)
+ public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc) throws CommunityException, CommunityNotFoundException {
+ if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) {
+ throw new CommunityException("parameters 'id' and zc.communityId must be coherent");
+ }
+ if(!StringUtils.isNotBlank(zc.getZenodoid())){
+ throw new CommunityException("parameter zenodoid cannot be null or empty");
+ }
+ final TreeMap zcs = getZenodoCommunityMap(id);
+
+ for(CommunityZenodoCommunity czc : zcs.values()){
+ if (czc.getZenodoid().equals(zc.getZenodoid())){
+ throw new CommunityException("Zenodo community already associated to the RCD");
+ }
+ }
+
+ zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
+
+ isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
+ cc.updateZenodoCommunity(id, zc);
+
+ return zc;
+ }
+
+ public CommunityOpenAIRECommunities getOpenAIRECommunities(String zenodoId) throws CommunityException, CommunityNotFoundException {
+
+ if(cci.getInverseZenodoCommunityMap().containsKey(zenodoId))
+ return new CommunityOpenAIRECommunities().setZenodoid(zenodoId).setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
+ return new CommunityOpenAIRECommunities();
+
+ }
+
+ // HELPERS
+
+ private TreeMap getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException {
+ return getCommunityProjects(id).stream()
+ .collect(Collectors.toMap(
+ p -> Integer.valueOf(p.getId()),
+ Functions.identity(),
+ (p1, p2) -> {
+ log.warn(String.format("duplicate project found: '%s'", p1.getId()));
+ return p2;
+ },
+ TreeMap::new));
+ }
+
+ private TreeMap getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException {
+ log.info("getting community content provider map");
+ return getCommunityContentproviders(id).stream()
+ .collect(Collectors.toMap(
+ cp -> Integer.valueOf(cp.getId()),
+ Functions.identity(),
+ (cp1, cp2) -> {
+ log.warn(String.format("duplicate content provider found: '%s'", cp1.getId()));
+ return cp2;
+ },
+ TreeMap::new));
+ }
+
+ private TreeMap getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException{
+ return getCommunityZenodoCommunities(id).stream()
+ .collect(Collectors.toMap(
+ cp -> Integer.valueOf(cp.getId()),
+ Functions.identity(),
+ (cp1, cp2) -> {
+ log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId()));
+ return cp2;
+ },
+ TreeMap::new));
+ }
+
+ private TreeMap getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException {
+ return getCommunityOrganizations(id).stream()
+ .collect(Collectors.toMap(
+ o -> Integer.valueOf(o.getId()),
+ Functions.identity(),
+ (o1, o2) -> {
+ log.warn(String.format("duplicate content provider found: '%s'", o1.getId()));
+ return o2;
+ },
+ TreeMap::new));
+ }
+
+ private Map getContextMap() throws CommunityException {
+ try {
+ return isClient.getCommunityContextMap();
+ } catch (IOException e) {
+ throw new CommunityException(e);
+ }
+ }
+
+ public CommunityOrganization addCommunityOrganization(String id, CommunityOrganization organization) throws CommunityException, CommunityNotFoundException {
+ if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) {
+ throw new CommunityException("parameters 'id' and organization.communityId must be coherent");
+ }
+
+ final TreeMap cps = getCommunityOrganizationMap(id);
+
+ final String organization_id = organization.getId();
+ if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))){
+ if (organization.getName() != null) {
+ isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME,organization.getName());
+ }
+ if(organization.getLogo_url()!= null){
+ isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()));
+ }
+ if (organization.getWebsite_url() != null){
+ isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()));
+ }
+ }else{
+ organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
+ isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization));
+ }
+
+
+ cc.updateOrganization(id, organization);
+ return organization;
+ }
+
+
+
+
+}
\ No newline at end of file
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java
new file mode 100644
index 00000000..dbc18fc5
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java
@@ -0,0 +1,12 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.Map;
+import java.util.Set;
+
+public interface CommunityClient {
+
+ Map> getInverseZenodoCommunityMap() throws CommunityException, CommunityNotFoundException;
+
+ void dropCache();
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java
new file mode 100644
index 00000000..96ac59a6
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java
@@ -0,0 +1,58 @@
+package eu.dnetlib.openaire.community;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+@Component
+public class CommunityClientImpl implements CommunityClient {
+
+ private static final Log log = LogFactory.getLog(CommunityClient.class);
+
+ @Autowired
+ private CommunityCommon communityCommon;
+
+ @Override
+ @Cacheable("community-cache")
+ public Map> getInverseZenodoCommunityMap () throws CommunityException, CommunityNotFoundException {
+ log.info("Creating the data structure. Not using cache");
+ final Map> inverseListMap = new HashMap<>();
+
+ final List communityList = communityCommon.listCommunities();
+
+ for(CommunitySummary cs :communityList){
+ final String communityId = cs.getId();
+ List czc = communityCommon.getCommunityZenodoCommunities(communityId);
+ for(CommunityZenodoCommunity zc:czc){
+ final String zenodoId = zc.getZenodoid();
+ if(!inverseListMap.containsKey(zenodoId)) {
+ inverseListMap.put(zc.getZenodoid(),new HashSet<>());
+ }
+ inverseListMap.get(zc.getZenodoid()).add(communityId);
+ }
+ final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
+ if(!inverseListMap.containsKey(zenodoMainCommunity)) {
+ inverseListMap.put(zenodoMainCommunity,new HashSet<>());
+ }
+
+ inverseListMap.get(zenodoMainCommunity).add(communityId);
+ }
+ return inverseListMap;
+ }
+
+
+
+ @Override
+ @CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true)
+ @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
+ public void dropCache(){
+ log.debug("dropped community cache");
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java
new file mode 100644
index 00000000..fde84333
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java
@@ -0,0 +1,438 @@
+package eu.dnetlib.openaire.community;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import eu.dnetlib.openaire.common.ISClient;
+import eu.dnetlib.openaire.context.Category;
+import eu.dnetlib.openaire.context.Concept;
+import eu.dnetlib.openaire.context.Context;
+import eu.dnetlib.openaire.context.Param;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static eu.dnetlib.openaire.community.CommunityConstants.*;
+
+@Component
+public class CommunityCommon {
+
+ @Autowired
+ private ISClient isClient;
+
+ public Map getContextMap() throws CommunityException {
+ try {
+ return isClient.getCommunityContextMap();
+ } catch (IOException e) {
+ throw new CommunityException(e);
+ }
+ }
+
+ public List listCommunities() throws CommunityException {
+ return getContextMap().values().stream()
+ .filter(context -> !communityBlackList.contains(context.getId()))
+ .map(CommunityMappingUtils::asCommunitySummary)
+ .collect(Collectors.toList());
+ }
+
+
+ public List getCommunityInfo(final String id, final String idSuffix, final Function mapping) throws CommunityException {
+ final Map contextMap = getContextMap();
+ final Context context = contextMap.get(id);
+ if (context != null) {
+ final Map categories = context.getCategories();
+ final Category category = categories.get(id + idSuffix);
+ if (category != null) {
+ return category.getConcepts().stream()
+ .map(mapping)
+ .collect(Collectors.toList());
+ }
+ }
+ return Lists.newArrayList();
+ }
+
+ public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
+ final Context context = getContextMap().get(id);
+ if (context == null || CommunityConstants.communityBlackList.contains(id)) {
+ throw new CommunityNotFoundException(String.format("community '%s' does not exist", id));
+ }
+ return CommunityMappingUtils.asCommunityProfile(context);
+ }
+
+ public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
+ getCommunity(id); // ensure the community exists.
+ return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
+ }
+
+
+ public void updateProject(String communityId, CommunityProject project) throws CommunityException {
+ final Context context = getContextMap().get(communityId);
+ Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX);
+ if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
+ .contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){
+ prj.getConcepts().forEach(concept -> {
+ if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) {
+ if (project.getName() != null) {
+
+ concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param()
+ .setName(CPROJECT_FULLNAME).setValue(project.getName())));
+ }
+ if (project.getAcronym() != null) {
+ if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){
+ concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param()
+ .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
+ }
+ else{
+ concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param()
+ .setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
+ }
+
+ }
+ if (project.getOpenaireId() != null) {
+ if(concept.getParams().keySet().contains(OPENAIRE_ID)){
+ concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
+ .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
+ }
+ else{
+ concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
+ .setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
+ }
+
+ }
+ if (project.getFunder() != null) {
+ concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param()
+ .setName(CPROJECT_FUNDER).setValue(project.getFunder())));
+ }
+ if (project.getGrantId() != null) {
+ concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param()
+ .setName(CPROJECT_NUMBER).setValue(project.getGrantId())));
+
+ }
+ }
+ });
+ }
+ else{
+ Concept concept = new Concept();
+ concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId());
+ concept.setClaim(false);
+ if(project.getAcronym() != null)
+ concept.setLabel(project.getAcronym());
+ else
+ concept.setLabel("");
+
+ Map> params = new TreeMap<>();
+
+ if(project.getAcronym() != null){
+ params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM)
+ .setValue(project.getAcronym())));
+ }
+
+ if (project.getName() != null){
+ params.put(CPROJECT_FULLNAME, Arrays.asList(new Param()
+ .setName(CPROJECT_FULLNAME)
+ .setValue(project.getName())
+ ));
+ }
+
+ if (project.getOpenaireId() != null){
+ params.put(OPENAIRE_ID, Arrays.asList(new Param()
+ .setName(OPENAIRE_ID)
+ .setValue(project.getOpenaireId())
+ ));
+ }
+
+ if(project.getFunder() != null){
+ params.put(CPROJECT_FUNDER, Arrays.asList(new Param()
+ .setName(CPROJECT_FUNDER)
+ .setValue(project.getFunder())
+ ));
+ }
+
+ if (project.getGrantId()!=null){
+ params.put(CPROJECT_NUMBER, Arrays.asList(new Param()
+ .setName(CPROJECT_NUMBER)
+ .setValue(project.getGrantId())
+ ));
+ }
+
+ concept.setParams(params);
+ prj.getConcepts().add(concept);
+
+
+ }
+
+ }
+
+ public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException {
+ final Context context = getContextMap().get(id);
+
+ if(community.getShortName() != null) {
+ context.setLabel(community.getShortName());
+ }
+
+ if (community.getName() != null){
+ context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param()
+ .setValue(community.getName()).setName(CSUMMARY_NAME)));
+ }
+ if(community.getDescription() != null) {
+ context.getParams()
+ .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
+ .setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription())));
+ }
+ if(community.getLogoUrl() != null){
+ context.getParams()
+ .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
+ .setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl())));
+
+ }
+ if (community.getStatus() != null) {
+ context.getParams()
+ .replace(CSUMMARY_STATUS, Arrays.asList(new Param()
+ .setName(CSUMMARY_STATUS).setValue(community.getStatus().name())));
+ }
+ if (community.getSubjects() != null) {
+ context.getParams()
+ .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
+ .setValue(Joiner.on(CSV_DELIMITER)
+ .join(community.getSubjects()))));
+ }
+ if(community.getMainZenodoCommunity() != null){
+ context.getParams()
+ .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
+ .setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity())));
+
+ }
+
+ }
+
+
+ public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException {
+ Map cmap = getContextMap();
+ Context context = cmap.get(communityId);
+ Map cat = context.getCategories();
+
+ List concepts = cat.get(communityId + category).getConcepts()
+ .stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList());
+
+ cat.get(communityId + category).setConcepts(concepts);
+ }
+
+ public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException {
+ final Context context = getContextMap().get(communityId);
+ Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX);
+ if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
+ .contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){
+ dts.getConcepts().forEach(concept -> {
+ if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) {
+
+
+ if (cp.getName() != null) {
+ if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){
+ concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
+ }
+ else{
+ concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
+ }
+
+ }
+ if (cp.getOfficialname() != null) {
+ if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){
+ concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
+ }
+ else{
+ concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
+ }
+
+ }
+
+
+ if (cp.getOpenaireId() != null) {
+ if(concept.getParams().keySet().contains(OPENAIRE_ID)){
+ concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
+ .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
+ }
+ else{
+ concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
+ .setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
+ }
+
+ }
+
+ if (cp.getSelectioncriteria() != null) {
+ if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){
+ concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
+ }
+ else{
+ concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
+ .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
+ }
+
+ }
+ }
+ });
+ }
+ else{
+ Concept concept = new Concept();
+ concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId());
+ concept.setClaim(false);
+ concept.setLabel("");
+
+ Map> params = new TreeMap<>();
+
+ if (cp.getName() != null) {
+ params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME)));
+ }
+ if(cp.getOfficialname()!= null){
+ params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME)));
+ }
+ if (cp.getOpenaireId() != null){
+ params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID)));
+ }
+ if(cp.getSelectioncriteria() != null){
+ params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA)));
+
+ }
+
+ concept.setParams(params);
+ dts.getConcepts().add(concept);
+
+
+ }
+
+ }
+
+ public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException {
+
+ final Context context = getContextMap().get(communityId);
+ Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX);
+ if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
+ .contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){
+ orgs.getConcepts().forEach(concept -> {
+ if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) {
+
+
+ if (organization.getName() != null) {
+ if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){
+ concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param()
+ .setName(CORGANIZATION_NAME).setValue(organization.getName())));
+ }
+ else{
+ concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param()
+ .setName(CORGANIZATION_NAME).setValue(organization.getName())));
+ }
+
+ }
+ if (organization.getLogo_url() != null) {
+ if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){
+ concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
+ .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
+ }
+ else{
+ concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
+ .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
+ }
+
+ }
+
+
+ if (organization.getWebsite_url() != null) {
+ if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){
+ concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
+ .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
+ }
+ else{
+ concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
+ .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
+ }
+
+ }
+
+ }
+
+ });
+ }
+ else{
+
+ Concept concept = new Concept();
+ concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId());
+ concept.setClaim(false);
+ concept.setLabel("");
+
+ Map> params = new TreeMap<>();
+
+ if (organization.getName() != null) {
+ params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME)));
+ }
+ if(organization.getLogo_url()!= null){
+
+ params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL)));
+ }
+ if (organization.getWebsite_url() != null){
+ params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL)));
+ }
+
+
+ concept.setParams(params);
+ orgs.getConcepts().add(concept);
+
+
+ }
+
+ }
+
+ public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException {
+ final Context context = getContextMap().get(communityId);
+ Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
+ if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
+ .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){
+ zcs.getConcepts().forEach(concept -> {
+ if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
+
+
+ if (zc.getZenodoid() != null) {
+ if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){
+ concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
+ .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
+ }
+ else{
+ concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
+ .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
+ }
+
+ }
+
+ }
+
+ });
+ }
+ else{
+
+ Concept concept = new Concept();
+ concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
+ concept.setClaim(false);
+
+
+ Map> params = new TreeMap<>();
+
+ if (zc.getZenodoid() != null) {
+ params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
+ concept.setLabel(zc.getZenodoid());
+ }else{
+ concept.setLabel("");
+ }
+ concept.setParams(params);
+ zcs.getConcepts().add(concept);
+
+ }
+
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java
new file mode 100644
index 00000000..9ac100f9
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java
@@ -0,0 +1,58 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+
+public class CommunityConstants {
+
+ public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
+
+ // common
+ public final static String OPENAIRE_ID = "openaireId";
+ public final static String PIPE_SEPARATOR = "||";
+ public final static String ID_SEPARATOR = "::";
+ public final static String CSV_DELIMITER = ",";
+ public final static String CLABEL = "label";
+
+ // id suffixes
+ public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
+ public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
+ public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
+ public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
+
+ // community summary
+ public final static String CSUMMARY_DESCRIPTION = "description";
+ public final static String CSUMMARY_LOGOURL = "logourl";
+ public final static String CSUMMARY_STATUS = "status";
+ public final static String CSUMMARY_NAME = "name";
+ public final static String CSUMMARY_MANAGER = "manager";
+ public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
+
+ // community profile
+ public final static String CPROFILE_SUBJECT = "subject";
+ public final static String CPROFILE_CREATIONDATE = "creationdate";
+
+ // community project
+ public final static String CPROJECT_FUNDER = "funder";
+ public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
+ public final static String CPROJECT_FULLNAME = "projectfullname";
+ public final static String CPROJECT_ACRONYM = "acronym";
+
+ // community content provider
+ public final static String CCONTENTPROVIDER_NAME = "name";
+ public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
+ public final static String CCONTENTPROVIDER_ENABLED = "enabled";
+ public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
+ public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
+
+ //community zenodo community
+ public final static String CZENODOCOMMUNITY_ID = "zenodoid";
+
+ //community organization
+ public final static String CORGANIZATION_NAME = "name";
+ public final static String CORGANIZATION_LOGOURL = "logourl";
+ public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java
new file mode 100644
index 00000000..9d470dc7
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java
@@ -0,0 +1,105 @@
+package eu.dnetlib.openaire.community;
+
+import javax.validation.constraints.NotNull;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.google.gson.Gson;
+import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+@JsonAutoDetect
+public class CommunityContentprovider {
+
+ private static final Log log = LogFactory.getLog(CommunityContentprovider.class);
+
+ @ApiModelProperty(value = "OpenAIRE identifier for this content provider, if available", required = false)
+ private String openaireId;
+
+ @NotNull
+ @ApiModelProperty(value = "the community identifier this content provider belongs to", required = true)
+ private String communityId;
+
+ @NotNull
+ @ApiModelProperty(value = "identifies this content provider within the context it belongs to", required = true)
+ private String id;
+
+ @ApiModelProperty(value = "content provider name", required = false)
+ private String name;
+
+ @NotNull
+ @ApiModelProperty(value = "content provider official name", required = true)
+ private String officialname;
+
+ //@NotNull
+ @ApiModelProperty(value = "content provider selection criteria", required = false)
+ private SelectionCriteria selectioncriteria;
+
+ public String getOpenaireId() {
+ return openaireId;
+ }
+
+ public void setOpenaireId(final String openaireId) {
+ this.openaireId = openaireId;
+ }
+
+ public String getCommunityId() {
+ return communityId;
+ }
+
+ public void setCommunityId(final String communityId) {
+ this.communityId = communityId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public String getOfficialname() {
+ return officialname;
+ }
+
+ public void setOfficialname(final String officialname) {
+ this.officialname = officialname;
+ }
+
+ public SelectionCriteria getSelectioncriteria() {
+
+ return this.selectioncriteria;
+ }
+
+ public void setSelectioncriteria(SelectionCriteria selectioncriteria) {
+ this.selectioncriteria = selectioncriteria;
+
+ }
+
+ public String toString(){
+ return String.format("id %s, name %s, selection criteria %s" , this.id, this.name, toJson());
+ }
+
+
+ public String toJson() {
+ if (selectioncriteria == null)
+ return "";
+ return new Gson().toJson(selectioncriteria);
+ }
+
+ public String toXML() {
+ if (selectioncriteria == null)
+ return "";
+ return "";
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java
new file mode 100644
index 00000000..5d315dd7
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java
@@ -0,0 +1,51 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.Date;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public class CommunityDetails extends CommunitySummary {
+
+ @ApiModelProperty("date of creation for this community")
+ private Date creationDate;
+
+ @ApiModelProperty("date of the last update for this communityu")
+ private Date lastUpdateDate;
+
+ @ApiModelProperty("list of subjects (keywords) that characterise this community")
+ private List subjects;
+
+ public CommunityDetails() {
+ }
+
+ public CommunityDetails(final CommunitySummary summary) {
+ super(summary);
+ }
+
+ public Date getCreationDate() {
+ return creationDate;
+ }
+
+ public void setCreationDate(final Date creationDate) {
+ this.creationDate = creationDate;
+ }
+
+ public List getSubjects() {
+ return subjects;
+ }
+
+ public void setSubjects(final List subjects) {
+ this.subjects = subjects;
+ }
+
+ public Date getLastUpdateDate() {
+ return lastUpdateDate;
+ }
+
+ public void setLastUpdateDate(Date lastUpdateDate) {
+ this.lastUpdateDate = lastUpdateDate;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java
new file mode 100644
index 00000000..9b2c2746
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java
@@ -0,0 +1,20 @@
+package eu.dnetlib.openaire.community;
+
+import java.io.IOException;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseBody
+@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
+public class CommunityException extends Exception {
+
+ public CommunityException(final String message) {
+ super(message);
+ }
+
+ public CommunityException(final IOException e) {
+ super(e);
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java
new file mode 100644
index 00000000..271c1cbe
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java
@@ -0,0 +1,231 @@
+package eu.dnetlib.openaire.community;
+
+import java.text.ParseException;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.escape.Escaper;
+import com.google.common.xml.XmlEscapers;
+import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
+import eu.dnetlib.openaire.context.Concept;
+import eu.dnetlib.openaire.context.Context;
+import eu.dnetlib.openaire.context.Param;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static eu.dnetlib.openaire.common.Utils.escape;
+import static eu.dnetlib.openaire.community.CommunityConstants.*;
+
+public class CommunityMappingUtils {
+
+ private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
+
+ private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
+
+ public static CommunitySummary asCommunitySummary(final Context c) {
+ final CommunitySummary summary = new CommunitySummary();
+
+ summary.setId(c.getId());
+ summary.setShortName(c.getLabel());
+ summary.setLastUpdateDate(c.getLastUpdateDate());
+ summary.setCreationDate(c.getCreationDate());
+ summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
+ summary.setType(c.getType());
+
+ final Map> params = c.getParams();
+ if (params.containsKey(CSUMMARY_DESCRIPTION)) {
+ summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
+ }
+ if (params.containsKey(CSUMMARY_LOGOURL)) {
+ summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
+ }
+ if (params.containsKey(CSUMMARY_STATUS)) {
+ summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
+ }
+ if (params.containsKey(CSUMMARY_NAME)) {
+ summary.setName(asCsv(params.get(CSUMMARY_NAME)));
+ }
+ if (params.containsKey(CSUMMARY_ZENODOC)) {
+ summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
+ }
+
+ return summary;
+ }
+
+ public static CommunityDetails asCommunityProfile(final Context c) {
+
+ final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
+ p.setLastUpdateDate(c.getLastUpdateDate());
+ final Map> params = c.getParams();
+ if (params.containsKey(CPROFILE_SUBJECT)) {
+ p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
+ }
+ if (params.containsKey(CPROFILE_CREATIONDATE)){
+ try {
+ p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern));
+ }catch(ParseException e) {
+ e.printStackTrace();
+ }
+ }
+
+ return p;
+ }
+
+ public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
+
+ final Map> p = c.getParams();
+ final CommunityProject project = new CommunityProject();
+ project.setCommunityId(communityId);
+ project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
+ project.setOpenaireId(firstValue(p, OPENAIRE_ID));
+ project.setFunder(firstValue(p, CPROJECT_FUNDER));
+ project.setGrantId(firstValue(p, CPROJECT_NUMBER));
+ project.setName(firstValue(p, CPROJECT_FULLNAME));
+ project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
+
+ return project;
+ }
+
+ public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
+
+ final Map> p = c.getParams();
+ final CommunityContentprovider d = new CommunityContentprovider();
+ d.setCommunityId(communityId);
+ d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
+ d.setOpenaireId(firstValue(p, OPENAIRE_ID));
+ d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
+ d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
+ d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
+ return d;
+ }
+
+ public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){
+ final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
+ final Map> p = c.getParams();
+ z.setCommunityId(communityId);
+ z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
+ z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID));
+ //z.setName(c.getLabel());
+ return z;
+ }
+
+
+ public static CommunityOrganization asCommunityOrganization(String id, Concept c) {
+ final Map> p = c.getParams();
+ final CommunityOrganization o = new CommunityOrganization();
+ o.setCommunityId(id);
+ o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
+ o.setName(firstValue(p,CORGANIZATION_NAME));
+ o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL)));
+ o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL)));
+
+ return o;
+ }
+
+ private static String getDecodedUrl(final String encoded_url){
+ if(encoded_url == null){
+ return encoded_url;
+ }
+ return new String(Base64.getDecoder().decode(encoded_url));
+ }
+
+
+ private static List splitValues(final Stream stream, final String separator) {
+ return stream.map(s -> s.split(separator))
+ .map(Arrays::asList)
+ .flatMap(List::stream)
+ .filter(StringUtils::isNotBlank)
+ .map(StringUtils::trim)
+ .collect(Collectors.toList());
+ }
+
+ private static String firstValue(final Map> p, final String paramName) {
+ return asValues(p.get(paramName)).findFirst().orElse(null);
+ }
+
+ private static String asCsv(final List params) {
+ return asValues(params)
+ .collect(Collectors.joining(CSV_DELIMITER));
+ }
+
+ private static Stream asValues(final List params) {
+ return params == null ? Stream.empty() : params.stream()
+ .map(Param::getValue)
+ .map(StringUtils::trim)
+ .distinct();
+ }
+
+ public static String asProjectXML(final String contextId, final CommunityProject project) {
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ final StringBuilder sb = new StringBuilder();
+ sb.append(
+ String.format(
+ "\n",
+ escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym())));
+ sb.append(paramXML(CPROJECT_FULLNAME, project.getName()));
+ sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym()));
+ sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId()));
+ sb.append(paramXML(CPROJECT_FUNDER, project.getFunder()));
+ sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId()));
+ sb.append("\n");
+ return sb.toString();
+ }
+
+ public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) {
+ log.info("creating the XML for the content provider");
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ final StringBuilder sb = new StringBuilder();
+ sb.append(
+ String.format(
+ "\n",
+ escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName())));
+ sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId()));
+ sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName()));
+ sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname()));
+ sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT));
+ sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML()));
+ sb.append("\n");
+ log.info(sb.toString());
+ return sb.toString();
+ }
+
+ public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ final StringBuilder sb = new StringBuilder();
+ sb.append(
+ String.format(
+ "\n",
+ escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
+
+ sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
+ sb.append("\n");
+ return sb.toString();
+ }
+
+
+ public static String asOrganizationXML(final String contextId, CommunityOrganization organization) {
+ final Escaper esc = XmlEscapers.xmlAttributeEscaper();
+ final StringBuilder sb = new StringBuilder();
+ sb.append(
+ String.format(
+ "\n",
+ escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName())));
+ sb.append(paramXML(CORGANIZATION_NAME, organization.getName()));
+ sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())));
+ sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())));
+ sb.append("\n");
+ return sb.toString();
+ }
+
+
+ private static String paramXML(final String paramName, final String value) {
+ return String.format("%s\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
+ }
+
+ private static String paramXMLNoEscape(final String paramName, final String value) {
+ return String.format("%s\n", paramName, value);
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java
new file mode 100644
index 00000000..3138d64a
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java
@@ -0,0 +1,19 @@
+package eu.dnetlib.openaire.community;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseBody
+@ResponseStatus(value = HttpStatus.NOT_FOUND)
+public class CommunityNotFoundException extends Exception {
+
+ public CommunityNotFoundException(final String msg) {
+ super(msg);
+ }
+
+ public CommunityNotFoundException(final Exception e) {
+ super(e);
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java
new file mode 100644
index 00000000..3cf3403f
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java
@@ -0,0 +1,42 @@
+package eu.dnetlib.openaire.community;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CommunityOpenAIRECommunities {
+
+ @NotNull
+ @ApiModelProperty(value = "the zenodo community identifier", required = true)
+ private String zenodoid;
+
+ @NotNull
+ @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true)
+ private List openAirecommunitylist;
+
+ public CommunityOpenAIRECommunities() {
+ this.zenodoid = "";
+ openAirecommunitylist=new ArrayList<>();
+ }
+
+ public List getOpenAirecommunitylist() {
+ return openAirecommunitylist;
+ }
+
+ public CommunityOpenAIRECommunities setOpenAirecommunitylist(List openAirecommunitylist) {
+ this.openAirecommunitylist = openAirecommunitylist;
+ return this;
+ }
+
+ public String getZenodoid() {
+ return zenodoid;
+ }
+
+ public CommunityOpenAIRECommunities setZenodoid(String zenodoid) {
+ this.zenodoid = zenodoid;
+ return this;
+ }
+
+}
\ No newline at end of file
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java
new file mode 100644
index 00000000..7ced9784
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java
@@ -0,0 +1,75 @@
+package eu.dnetlib.openaire.community;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+
+import io.swagger.annotations.ApiModelProperty;
+import javax.validation.constraints.NotNull;
+
+@JsonAutoDetect
+public class CommunityOrganization {
+ @NotNull
+ @ApiModelProperty(value = "the community identifier this organization belongs to", required = true)
+ private String communityId;
+
+ @NotNull
+ @ApiModelProperty(value = "name of the organization", required = true)
+ private String name;
+
+ @NotNull
+ @ApiModelProperty(value = "identifies this organization within the context it belongs to", required = true)
+ private String id;
+
+ @NotNull
+ @ApiModelProperty(value="url of the logo for this organization", required = true)
+ private String logo_url;
+
+
+ @NotNull
+ @ApiModelProperty(value="website url for this organization", required = true)
+ private String website_url;
+
+ public String getCommunityId() {
+ return communityId;
+ }
+
+ public CommunityOrganization setCommunityId(String communityId) {
+ this.communityId = communityId;
+ return this;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public CommunityOrganization setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public CommunityOrganization setId(String id) {
+ this.id = id;
+ return this;
+ }
+
+ public String getLogo_url() {
+ return logo_url;
+ }
+
+ public CommunityOrganization setLogo_url(String logo_url) {
+ this.logo_url = logo_url;
+ return this;
+ }
+
+ public String getWebsite_url() {
+ return website_url;
+ }
+
+ public CommunityOrganization setWebsite_url(String website_url) {
+ this.website_url = website_url;
+ return this;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java
new file mode 100644
index 00000000..205e844a
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java
@@ -0,0 +1,86 @@
+package eu.dnetlib.openaire.community;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public class CommunityProject {
+
+ @ApiModelProperty(value = "OpenAIRE identifier for this project, if available", required = false)
+ private String openaireId;
+
+ @ApiModelProperty(value = "the community identifier this project belongs to", required = true)
+ private String communityId;
+
+ @ApiModelProperty(value = "identifies this project within the context it belongs to", required = true)
+ private String id;
+
+ @ApiModelProperty(value = "project name", required = true)
+ private String name;
+
+ @ApiModelProperty(value = "project acronym", required = false)
+ private String acronym;
+
+ @ApiModelProperty(value = "project funder", required = true)
+ private String funder;
+
+ @ApiModelProperty(value = "project grant id", required = true)
+ private String grantId;
+
+ public String getOpenaireId() {
+ return openaireId;
+ }
+
+ public void setOpenaireId(final String openaireId) {
+ this.openaireId = openaireId;
+ }
+
+ public String getCommunityId() {
+ return communityId;
+ }
+
+ public void setCommunityId(final String communityId) {
+ this.communityId = communityId;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public String getAcronym() {
+ return acronym;
+ }
+
+ public void setAcronym(final String acronym) {
+ this.acronym = acronym;
+ }
+
+ public String getFunder() {
+ return funder;
+ }
+
+ public void setFunder(final String funder) {
+ this.funder = funder;
+ }
+
+ public String getGrantId() {
+ return grantId;
+ }
+
+ public void setGrantId(final String grantId) {
+ this.grantId = grantId;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java
new file mode 100644
index 00000000..e3b4902a
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java
@@ -0,0 +1,17 @@
+package eu.dnetlib.openaire.community;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public enum CommunityStatus {
+
+ @ApiModelProperty("restricted visibility")
+ hidden,
+
+ @ApiModelProperty("visible only to RCD managers")
+ manager,
+
+ @ApiModelProperty("visible to RCD managers and to the community users")
+ all
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java
new file mode 100644
index 00000000..c3453e0e
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java
@@ -0,0 +1,167 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.Date;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public class CommunitySummary {
+
+ @ApiModelProperty("identifies the community")
+ protected String id;
+
+ @ApiModelProperty("values for this field reflect the index field _community_ in the index, e.g. 'egi||EGI Federation'")
+ protected String queryId;
+
+ @ApiModelProperty("community type")
+ protected String type;
+
+ @ApiModelProperty("community name")
+ protected String name;
+
+ @ApiModelProperty("community short name")
+ protected String shortName;
+
+ @ApiModelProperty("community creation date")
+ protected Date creationDate;
+
+ @ApiModelProperty("community last update date")
+ protected Date lastUpdateDate;
+
+ @ApiModelProperty("community description")
+ protected String description;
+
+ @ApiModelProperty("http url for the community logo")
+ protected String logoUrl;
+
+ @ApiModelProperty("status of the community, drives its visibility")
+ protected CommunityStatus status;
+
+ @ApiModelProperty("Zenodo community associated to this community")
+ protected String zenodoCommunity;
+
+ public CommunitySummary() {
+ }
+
+ public CommunitySummary(
+ final String id,
+ final String queryId,
+ final String type,
+ final String name,
+ final String shortName,
+ final Date creationDate,
+ final Date lastUpdateDate,
+ final String description,
+ final String logoUrl,
+ final CommunityStatus status,
+ final String zenodoCommunity) {
+ this.id = id;
+ this.queryId = queryId;
+ this.type = type;
+ this.name = name;
+ this.shortName = shortName;
+ this.creationDate = creationDate;
+ this.lastUpdateDate = lastUpdateDate;
+ this.description = description;
+ this.logoUrl = logoUrl;
+ this.status = status;
+ this.zenodoCommunity = zenodoCommunity;
+ }
+
+ public CommunitySummary(final CommunitySummary summary) {
+ this(summary.getId(),
+ summary.getQueryId(),
+ summary.getType(),
+ summary.getName(),
+ summary.getShortName(),
+ summary.getCreationDate(),
+ summary.getLastUpdateDate(),
+ summary.getDescription(),
+ summary.getLogoUrl(),
+ summary.getStatus(),
+ summary.getZenodoCommunity());
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(final String queryId) {
+ this.queryId = queryId;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(final String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public String getShortName() {
+ return shortName;
+ }
+
+ public void setShortName(final String shortName) {
+ this.shortName = shortName;
+ }
+
+ public Date getCreationDate() { return creationDate; }
+
+ public void setCreationDate(final Date creationDate) { this.creationDate = creationDate; }
+
+ public Date getLastUpdateDate() { return lastUpdateDate; }
+
+ public void setLastUpdateDate(final Date lastUpdateDate) { this.lastUpdateDate = lastUpdateDate; }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(final String description) {
+ this.description = description;
+ }
+
+ public String getLogoUrl() {
+ return logoUrl;
+ }
+
+ public void setLogoUrl(final String logoUrl) {
+ this.logoUrl = logoUrl;
+ }
+
+ public CommunityStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(final CommunityStatus status) {
+ this.status = status;
+ }
+
+ public String getZenodoCommunity() {
+ return zenodoCommunity;
+ }
+
+ public void setZenodoCommunity(String zenodoCommunity) {
+ this.zenodoCommunity = zenodoCommunity;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java
new file mode 100644
index 00000000..08ca7a26
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java
@@ -0,0 +1,96 @@
+package eu.dnetlib.openaire.community;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public class CommunityWritableProperties {
+
+ @ApiModelProperty("community name")
+ private String name;
+
+ @ApiModelProperty("community short name")
+ private String shortName;
+
+ @ApiModelProperty("community description")
+ private String description;
+
+ @ApiModelProperty("http url for the community logo")
+ private String logoUrl;
+
+ @ApiModelProperty("list of subjects (keywords) that characterise this community")
+ private List subjects;
+
+ @ApiModelProperty("status of the community, drives its visibility")
+ private CommunityStatus status;
+
+ @ApiModelProperty("id of the main Zenodo community")
+ private String mainZenodoCommunity;
+
+
+ public static CommunityWritableProperties fromDetails(final CommunityDetails details) {
+ CommunityWritableProperties p = new CommunityWritableProperties();
+ p.setName(details.getName());
+ p.setShortName(details.getShortName());
+ p.setDescription(details.getDescription());
+ p.setLogoUrl(details.getLogoUrl());
+ p.setSubjects(details.getSubjects());
+ p.setStatus(details.getStatus());
+ p.setMainZenodoCommunity(details.getZenodoCommunity());
+ return p;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ public String getShortName() {
+ return shortName;
+ }
+
+ public void setShortName(final String shortName) {
+ this.shortName = shortName;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(final String description) {
+ this.description = description;
+ }
+
+ public String getLogoUrl() {
+ return logoUrl;
+ }
+
+ public void setLogoUrl(final String logoUrl) {
+ this.logoUrl = logoUrl;
+ }
+
+ public List getSubjects() {
+ return subjects;
+ }
+
+ public void setSubjects(final List subjects) {
+ this.subjects = subjects;
+ }
+
+ public CommunityStatus getStatus() {
+ return status;
+ }
+
+ public void setStatus(final CommunityStatus status) {
+ this.status = status;
+ }
+
+ public String getMainZenodoCommunity() { return mainZenodoCommunity; }
+
+ public void setMainZenodoCommunity(String mainZenodoCommunity) { this.mainZenodoCommunity = mainZenodoCommunity; }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java
new file mode 100644
index 00000000..c84c1692
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java
@@ -0,0 +1,47 @@
+package eu.dnetlib.openaire.community;
+
+import javax.validation.constraints.NotNull;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import io.swagger.annotations.ApiModelProperty;
+
+@JsonAutoDetect
+public class CommunityZenodoCommunity {
+
+ @NotNull
+ @ApiModelProperty(value = "the community identifier this zenodo Community belongs to", required = true)
+ private String communityId;
+
+ @NotNull
+ @ApiModelProperty(value = "Zenodo identifier for this community", required = true)
+ private String zenodoid;
+
+ @NotNull
+ @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true)
+ private String id;
+
+ public String getZenodoid() {
+ return zenodoid;
+ }
+
+ public void setZenodoid(String zenodoid) {
+ this.zenodoid = zenodoid;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getCommunityId() {
+ return communityId;
+ }
+
+ public void setCommunityId(String communityId) {
+ this.communityId = communityId;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java
new file mode 100644
index 00000000..9f31965c
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java
@@ -0,0 +1,44 @@
+package eu.dnetlib.openaire.community.selectioncriteria;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+
+import java.io.Serializable;
+
+
+@JsonAutoDetect
+public class Constraint implements Serializable {
+ private String verb;
+ private String field;
+ private String value;
+
+
+ public Constraint() {
+ }
+
+ public String getVerb() {
+ return verb;
+ }
+
+ public void setVerb(String verb) {
+ this.verb = verb;
+ }
+
+ public String getField() {
+ return field;
+ }
+
+ public void setField(String field) {
+ this.field = field;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java
new file mode 100644
index 00000000..863d97c8
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java
@@ -0,0 +1,23 @@
+package eu.dnetlib.openaire.community.selectioncriteria;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+
+import java.io.Serializable;
+import java.util.List;
+
+@JsonAutoDetect
+public class Constraints implements Serializable {
+
+ private List constraint;
+
+
+ public Constraints() {
+ }
+ public List getConstraint() {
+ return constraint;
+ }
+
+ public void setConstraint(List constraint) {
+ this.constraint = constraint;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java
new file mode 100644
index 00000000..3b0c09f1
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java
@@ -0,0 +1,30 @@
+package eu.dnetlib.openaire.community.selectioncriteria;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.google.gson.Gson;
+import java.io.Serializable;
+import java.util.List;
+
+@JsonAutoDetect
+public class SelectionCriteria implements Serializable {
+ private List criteria;
+
+ public SelectionCriteria() {
+ }
+
+
+ public List getCriteria() {
+ return criteria;
+ }
+
+ public void setCriteria(List criteria) {
+ this.criteria = criteria;
+ }
+
+
+
+ public static SelectionCriteria fromJson(final String json) {
+ return new Gson().fromJson(json, SelectionCriteria.class);
+
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java
new file mode 100644
index 00000000..1c029777
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java
@@ -0,0 +1,67 @@
+package eu.dnetlib.openaire.context;
+
+import java.util.List;
+import java.util.Map;
+
+public class Category {
+
+ private String id;
+
+ private String label;
+
+ private boolean claim;
+
+ private Map> params;
+
+ private List concepts;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public boolean isClaim() {
+ return claim;
+ }
+
+ public boolean hasConcepts() {
+ return getConcepts() != null && !getConcepts().isEmpty();
+ }
+
+ public Map> getParams() {
+ return params;
+ }
+
+ public List getConcepts() {
+ return concepts;
+ }
+
+ public Category setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Category setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public Category setClaim(final boolean claim) {
+ this.claim = claim;
+ return this;
+ }
+
+ public Category setParams(final Map> params) {
+ this.params = params;
+ return this;
+ }
+
+ public Category setConcepts(final List concepts) {
+ this.concepts = concepts;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java
new file mode 100644
index 00000000..c2acb200
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java
@@ -0,0 +1,38 @@
+package eu.dnetlib.openaire.context;
+
+public class CategorySummary {
+
+ private String id;
+
+ private String label;
+
+ private boolean hasConcept;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public boolean isHasConcept() {
+ return hasConcept;
+ }
+
+ public CategorySummary setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public CategorySummary setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public CategorySummary setHasConcept(final boolean hasConcept) {
+ this.hasConcept = hasConcept;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java
new file mode 100644
index 00000000..ddbcdec1
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java
@@ -0,0 +1,67 @@
+package eu.dnetlib.openaire.context;
+
+import java.util.List;
+import java.util.Map;
+
+public class Concept {
+
+ private String id;
+
+ private String label;
+
+ private boolean claim;
+
+ private Map> params;
+
+ private List concepts;
+
+ public boolean hasSubConcepts() {
+ return getConcepts() != null && !getConcepts().isEmpty();
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public boolean isClaim() {
+ return claim;
+ }
+
+ public Map> getParams() {
+ return params;
+ }
+
+ public List getConcepts() {
+ return concepts;
+ }
+
+ public Concept setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Concept setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public Concept setClaim(final boolean claim) {
+ this.claim = claim;
+ return this;
+ }
+
+ public Concept setParams(final Map> params) {
+ this.params = params;
+ return this;
+ }
+
+ public Concept setConcepts(final List concepts) {
+ this.concepts = concepts;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java
new file mode 100644
index 00000000..58600836
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java
@@ -0,0 +1,51 @@
+package eu.dnetlib.openaire.context;
+
+import java.util.List;
+
+public class ConceptSummary {
+
+ private String id;
+
+ private String label;
+
+ public boolean hasSubConcept;
+
+ private List concepts;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public List getConcepts() {
+ return concepts;
+ }
+
+ public ConceptSummary setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public ConceptSummary setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public boolean isHasSubConcept() {
+ return hasSubConcept;
+ }
+
+ public ConceptSummary setHasSubConcept(final boolean hasSubConcept) {
+ this.hasSubConcept = hasSubConcept;
+ return this;
+ }
+
+ public ConceptSummary setConcept(final List concepts) {
+ this.concepts = concepts;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java
new file mode 100644
index 00000000..badc1ff1
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java
@@ -0,0 +1,91 @@
+package eu.dnetlib.openaire.context;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class Context {
+
+ private String id;
+
+ private String label;
+
+ private String type;
+
+ private Date creationDate;
+
+ private Date lastUpdateDate;
+
+ private Map> params;
+
+ private Map categories;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public Date getCreationDate() {
+ return creationDate;
+ }
+
+ public Map> getParams() {
+ return params;
+ }
+
+ public Map getCategories() {
+ return categories;
+ }
+
+
+
+ public Context setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Context setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public Context setType(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Context setCreationDate(final Date dateofcreation) {
+ this.creationDate = dateofcreation;
+ return this;
+ }
+
+
+ public Date getLastUpdateDate() {
+ return lastUpdateDate;
+ }
+
+ public Context setLastUpdateDate(Date lastUpdateDate) {
+ this.lastUpdateDate = lastUpdateDate;
+ return this;
+ }
+
+
+ public Context setParams(final Map> params) {
+ this.params = params;
+ return this;
+ }
+
+ public Context setCategories(final Map categories) {
+ this.categories = categories;
+ return this;
+ }
+
+}
+
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java
new file mode 100644
index 00000000..64aae1fe
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java
@@ -0,0 +1,86 @@
+package eu.dnetlib.openaire.context;
+
+import java.util.List;
+import java.util.Optional;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@CrossOrigin(origins = { "*" })
+@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+@io.swagger.annotations.Api(tags = "OpenAIRE Context API", description = "the OpenAIRE Context API")
+public class ContextApiController {
+
+ @Autowired
+ private ContextApiCore contextApiCore;
+
+ @RequestMapping(value = "/contexts", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "list brief information about all the context profiles",
+ notes = "list brief information about all the context profiles.",
+ tags = { },
+ response = ContextSummary[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = ContextSummary[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
+ public List listContexts(@RequestParam(required = false, defaultValue = "") List type) throws ContextException {
+ return contextApiCore.listContexts(type);
+ }
+
+ @RequestMapping(value = "/context/{contextId}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "list the categories defined within a context",
+ notes = "list the categories defined within a context",
+ tags = { },
+ response = CategorySummary[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = CategorySummary[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
+ public List listCategories(
+ @PathVariable final String contextId,
+ @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+
+ Boolean allFilter = Optional.ofNullable(all).orElse(false);
+ return contextApiCore.listCategories(contextId, allFilter);
+ }
+
+ @RequestMapping(value = "/context/category/{categoryId}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "list the concepts defined within a category",
+ notes = "list the concepts defined within a category",
+ tags = { },
+ response = ConceptSummary[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
+ public List listConcepts(
+ @PathVariable final String categoryId,
+ @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+
+ Boolean allFilter = Optional.ofNullable(all).orElse(false);
+ return contextApiCore.listConcepts(categoryId, allFilter);
+ }
+
+ @RequestMapping(value = "/context/category/concept/{conceptId}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(
+ value = "list the concepts defined within a category",
+ notes = "list the concepts defined within a category",
+ tags = { },
+ response = ConceptSummary[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
+ public List listSubConcepts(
+ @PathVariable final String conceptId,
+ @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
+
+ Boolean allFilter = Optional.ofNullable(all).orElse(false);
+ return contextApiCore.listSubConcepts(conceptId, allFilter);
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java
new file mode 100644
index 00000000..95dea205
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java
@@ -0,0 +1,120 @@
+package eu.dnetlib.openaire.context;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import eu.dnetlib.openaire.common.ISClient;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
+public class ContextApiCore {
+
+ private static final Log log = LogFactory.getLog(ContextApiCore.class);
+ private static final String SEPARATOR = "::";
+
+ @Autowired
+ private ISClient isClient;
+
+ public List listContexts(final List type) throws ContextException {
+ return getContextMap(type).values().stream()
+ .map(c -> new ContextSummary()
+ .setId(c.getId())
+ .setType(c.getType())
+ .setLabel(c.getLabel())
+ .setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : ""))
+ .collect(Collectors.toList());
+ }
+
+ public List listCategories(final String contextId, Boolean all) throws ContextException {
+ final Stream categories = getContextMap().get(contextId).getCategories().values().stream();
+ return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim));
+ }
+
+ private List asCategorySummaries(Stream categories) {
+ return categories
+ .map(c -> new CategorySummary()
+ .setId(c.getId())
+ .setLabel(c.getLabel())
+ .setHasConcept(c.hasConcepts()))
+ .collect(Collectors.toList());
+ }
+
+ public List listConcepts(final String categoryId, Boolean all) throws ContextException {
+ final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR);
+ final Stream concepts = getContextMap().get(contextId)
+ .getCategories()
+ .get(categoryId)
+ .getConcepts()
+ .stream();
+
+ return all ? asConceptSummaries(concepts) : asConceptSummaries(concepts.filter(Concept::isClaim));
+ }
+
+ private List asConceptSummaries(Stream concepts) {
+ return concepts
+ .map(c -> new ConceptSummary()
+ .setId(c.getId())
+ .setLabel(c.getLabel())
+ .setHasSubConcept(c.hasSubConcepts()))
+ .collect(Collectors.toList());
+ }
+
+ public List listSubConcepts(final String conceptId, Boolean all) throws ContextException {
+ final List ids = Splitter.on(SEPARATOR).splitToList(conceptId);
+ if (ids.size() < 3) {
+ throw new ContextException("");
+ }
+
+ final String contextId = ids.get(0);
+ final String categoryId = contextId + SEPARATOR + ids.get(1);
+
+ final Stream concepts = getContextMap().get(contextId)
+ .getCategories()
+ .get(categoryId)
+ .getConcepts()
+ .stream()
+ .filter(c -> conceptId.equals(c.getId()));
+
+ return all ?
+ mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) :
+ mapConcepts(concepts.collect(Collectors.toList()));
+ }
+
+ private List mapConcepts(final List concepts) {
+ if (concepts == null || concepts.isEmpty()) {
+ return null;
+ }
+ return concepts.stream()
+ .map(c -> new ConceptSummary()
+ .setId(c.getId())
+ .setLabel(c.getLabel())
+ .setHasSubConcept(c.hasSubConcepts())
+ .setConcept(mapConcepts(c.getConcepts())))
+ .collect(Collectors.toList());
+ }
+
+ private Map getContextMap() throws ContextException {
+ return getContextMap(Lists.newArrayList());
+ }
+
+ private Map getContextMap(final List type) throws ContextException {
+ try {
+ return isClient.getContextMap(type);
+ } catch (IOException e) {
+ throw new ContextException(e);
+ }
+ }
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java
new file mode 100644
index 00000000..be1fb392
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java
@@ -0,0 +1,20 @@
+package eu.dnetlib.openaire.context;
+
+import java.io.IOException;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseBody
+@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
+public class ContextException extends Exception {
+
+ public ContextException(final String message) {
+ super(message);
+ }
+
+ public ContextException(final IOException e) {
+ super(e);
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java
new file mode 100644
index 00000000..c3537343
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java
@@ -0,0 +1,113 @@
+package eu.dnetlib.openaire.context;
+
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+import org.dom4j.Node;
+
+import com.google.common.base.Functions;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.openaire.funders.domain.FunderDetails;
+
+public class ContextMappingUtils {
+
+ private static final List DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
+
+ public static Context parseContext(final String s, final Queue errors) {
+ try {
+ final Document doc = DocumentHelper.parseText(s);
+ final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context");
+ final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()");
+ final Context c = new Context()
+ .setId(eContext.attributeValue("id"))
+ .setLabel(eContext.attributeValue("label"))
+ .setType(eContext.attributeValue("type"))
+ .setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value")))
+ .setParams(parseParams(eContext))
+ .setCategories(parseCategories(eContext));
+ // the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check.
+ if (StringUtils.isNotBlank(creationDate)) {
+ c.setCreationDate(asDate(creationDate));
+ }
+ return c;
+ } catch (final DocumentException e) {
+ errors.add(e);
+ return new Context();
+ }
+ }
+
+ private static Date asDate(final String s) {
+ for (final String pattern : DATE_PATTERN) {
+ try {
+ return DateUtils.parseDate(s, pattern);
+ } catch (final ParseException e) {}
+ }
+ return null;
+ }
+
+ private static Map parseCategories(final Element eContext) {
+ final List eCategory = eContext.selectNodes("//category");
+ return eCategory.stream()
+ .map(n -> (Element) n)
+ .map(eCat -> new Category()
+ .setClaim(getClaim(eCat))
+ .setId(eCat.attributeValue("id"))
+ .setLabel(eCat.attributeValue("label"))
+ .setParams(parseParams(eCat))
+ .setConcepts(parseConcepts(eCat)))
+ .collect(Collectors.toMap(Category::getId, Functions.identity()));
+ }
+
+ private static List parseConcepts(final Element eCategory) {
+ final List eConcepts = eCategory.selectNodes("./concept");
+ return eConcepts.stream()
+ .map(n -> (Element) n)
+ .map(eCon -> new Concept()
+ .setClaim(getClaim(eCon))
+ .setId(eCon.attributeValue("id"))
+ .setLabel(eCon.attributeValue("label"))
+ .setParams(parseParams(eCon))
+ .setConcepts(parseConcepts(eCon)))
+ .collect(Collectors.toList());
+ }
+
+ private static Boolean getClaim(final Element eCon) {
+ final String claim = eCon.attributeValue("claim");
+ return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
+ }
+
+ private static Map> parseParams(final Element e) {
+ final List params = e.selectNodes("./param");
+ return params.stream()
+ .map(n -> (Element) n)
+ .map(p -> new Param()
+ .setName(p.attributeValue("name"))
+ .setValue(p.getTextTrim()))
+ .collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
+ final List p = new ArrayList<>(p1);
+ p.addAll(p2);
+ return p;
+ }));
+ }
+
+ public static FunderDetails asFunderDetails(final Context c) {
+ return new FunderDetails()
+ .setId(c.getId())
+ .setName(c.getLabel())
+ .setShortname(c.getId());
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java
new file mode 100644
index 00000000..212b5334
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java
@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.context;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseBody
+@ResponseStatus(value = HttpStatus.NOT_FOUND)
+public class ContextNotFoundException extends Exception {
+
+ public ContextNotFoundException(final String msg) {
+ super(msg);
+ }
+
+ public ContextNotFoundException(final Exception e) {
+ super(e);
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java
new file mode 100644
index 00000000..9e974a3d
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java
@@ -0,0 +1,49 @@
+package eu.dnetlib.openaire.context;
+
+public class ContextSummary {
+
+ private String id;
+
+ private String label;
+
+ private String type;
+
+ private String status;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public ContextSummary setId(final String id) {
+ this.id = id;
+ return this;
+ }
+
+ public ContextSummary setLabel(final String label) {
+ this.label = label;
+ return this;
+ }
+
+ public ContextSummary setType(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ public ContextSummary setStatus(final String status) {
+ this.status = status;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java
new file mode 100644
index 00000000..e83a6fd2
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java
@@ -0,0 +1,27 @@
+package eu.dnetlib.openaire.context;
+
+public class Param {
+
+ private String name;
+
+ private String value;
+
+ public String getName() {
+ return name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public Param setName(final String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Param setValue(final String value) {
+ this.value = value;
+ return this;
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java
new file mode 100755
index 00000000..c380a6cf
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java
@@ -0,0 +1,498 @@
+package eu.dnetlib.openaire.dsm;
+
+import static eu.dnetlib.openaire.common.ExporterConstants.API;
+import static eu.dnetlib.openaire.common.ExporterConstants.D;
+import static eu.dnetlib.openaire.common.ExporterConstants.DS;
+import static eu.dnetlib.openaire.common.ExporterConstants.M;
+import static eu.dnetlib.openaire.common.ExporterConstants.R;
+import static eu.dnetlib.openaire.common.ExporterConstants.W;
+
+import java.util.List;
+
+import javax.validation.Valid;
+
+import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.openaire.dsm.domain.*;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.http.HttpStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
+import eu.dnetlib.openaire.common.AbstractExporterController;
+import eu.dnetlib.openaire.common.OperationManager;
+import eu.dnetlib.openaire.vocabularies.Country;
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+
+@RestController
+@CrossOrigin(origins = { "*" })
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+@io.swagger.annotations.Api(tags = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API")
+public class DsmApiController extends AbstractExporterController {
+
+ @Autowired
+ private DsmCore dsmCore;
+
+ @RequestMapping(value = "/ds/countries", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = { DS, R }, response = Country[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = Country[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public List listCountries() throws DsmException {
+ return dsmCore.listCountries();
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/search/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceSearchResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public DatasourceSearchResponse search(
+ @RequestParam final RequestSort requestSortBy,
+ @RequestParam final RequestSortOrder order,
+ @RequestBody final RequestFilter requestFilter,
+ @PathVariable final int page,
+ @PathVariable final int size) throws DsmException {
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSearchResponse rsp = dsmCore.search(requestSortBy, order, requestFilter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = DatasourceDetailResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public DatasourceDetailResponse searchDsDetails(
+ @RequestParam final RequestSort requestSortBy,
+ @RequestParam final RequestSortOrder order,
+ @RequestBody final RequestFilter requestFilter,
+ @PathVariable final int page,
+ @PathVariable final int size) throws DsmException {
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { DS, R }, response = AggregationHistoryResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException {
+ final StopWatch stop = StopWatch.createStarted();
+ final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId);
+ return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = { DS, R }, response = DatasourceSnippetResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public DatasourceSnippetResponse searchSnippet(
+ @RequestParam final RequestSort requestSortBy,
+ @RequestParam final RequestSortOrder order,
+ @RequestBody final RequestFilter requestFilter,
+ @PathVariable final int page,
+ @PathVariable final int size) throws DsmException {
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = { DS,
+ R }, response = DatasourceSnippetResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public DatasourceSnippetResponse searchRegistered(
+ @RequestParam final RequestSort requestSortBy,
+ @RequestParam final RequestSortOrder order,
+ @RequestBody final RequestFilter requestFilter,
+ @PathVariable final int page,
+ @PathVariable final int size) throws DsmException {
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/recentregistered/{size}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = { DS,
+ R }, response = SimpleResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable {
+ final StopWatch stop = StopWatch.createStarted();
+ final SimpleResponse rsp = dsmCore.searchRecentRegistered(size);
+ return prepareResponse(1, size, stop, rsp);
+ }
+
+ @RequestMapping(value = "/ds/countregistered", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = { DS,
+ R }, response = Long.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = Long.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public Long countRegistered(@RequestParam final String fromDate,
+ @RequestParam(required = false) final String typologyFilter) throws Throwable {
+ return dsmCore.countRegisteredAfter(fromDate, typologyFilter);
+ }
+
+ @RequestMapping(value = "/ds/api/{dsId}", produces = { "application/json" }, method = RequestMethod.GET)
+ @ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = { API,
+ R }, response = ApiDetailsResponse.class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public ApiDetailsResponse getApi(
+ @PathVariable final String dsId) throws DsmException {
+
+ final StopWatch stop = StopWatch.createStarted();
+ final ApiDetailsResponse rsp = dsmCore.getApis(dsId);
+ return prepareResponse(0, rsp.getApi().size(), stop, rsp);
+ }
+
+ @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST)
+ @ApiOperation(value = "search for the list of base URLs of Datasource APIs managed by a user", notes = "Returns the list of base URLs of Datasource APIs managed by a user", tags = {
+ DS, API, R }, response = String[].class)
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = String[].class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public List searchBaseUrls(
+ @RequestBody final RequestFilter requestFilter,
+ @PathVariable final int page,
+ @PathVariable final int size) throws DsmException {
+
+ return dsmCore.findBaseURLs(requestFilter, page, size);
+ }
+
+ @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE)
+ @ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = { API, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class),
+ @ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class),
+ @ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class)})
+ public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException {
+ dsmCore.deleteApi(apiId);
+ }
+
+ @RequestMapping(value = "/ds/manage", method = RequestMethod.POST)
+ @ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = { DS, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void setManaged(
+ @RequestParam final String id,
+ @RequestParam final boolean managed) throws DsmException {
+
+ dsmCore.setManaged(id, managed);
+ }
+
+ @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET)
+ @ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = { DS, R })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public boolean isManaged(@PathVariable final String id) throws DsmException {
+ return dsmCore.isManaged(id);
+ }
+
+ @RequestMapping(value = "/ds/add", method = RequestMethod.POST)
+ @ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = { DS, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class),
+ @ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class) })
+ public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException {
+
+ if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API
+ throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", datasource.getId()));
+ }
+ dsmCore.save(datasource);
+ }
+
+ @RequestMapping(value = "/ds/update", method = RequestMethod.POST)
+ @ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = { DS, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateDatasource(
+ @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException {
+
+ dsmCore.updateDatasource(ds);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/logourl", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateDatasourceLogoURL(
+ @RequestParam final String dsId,
+ @RequestParam final String logourl) throws DsmException {
+
+ dsmCore.updateDatasourceLogoUrl(dsId, logourl);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/name", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource name", notes = "update a datasource name", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateDatasourceName(
+ @RequestParam final String dsId,
+ @RequestParam final String officialname,
+ @RequestParam final String englishname) throws DsmException {
+
+ dsmCore.updateDatasourcename(dsId, officialname, englishname);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/coordinates", method = RequestMethod.POST)
+ @ApiOperation(value = "update the datasource coordinates (latitude, longitude)", notes = "update the datasource coordinates (latitude, longitude)", tags = {
+ DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateCoordinates(
+ @RequestParam final String dsId,
+ @RequestParam final Double latitude,
+ @RequestParam final Double longitude) throws DsmException {
+
+ dsmCore.updateCoordinates(dsId, latitude, longitude);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/timezone", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource timezone", notes = "update a datasource timezone", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateTimezone(
+ @RequestParam final String dsId,
+ @RequestParam final String timezone) throws DsmException {
+
+ dsmCore.updateTimezone(dsId, timezone);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/typology", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource typology code", notes = "update a datasource typology code", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateTypology(
+ @RequestParam final String dsId,
+ @RequestParam final String typology) throws DsmException {
+
+ dsmCore.updateDsTypology(dsId, typology);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/registeredby", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource registeredBy", notes = "update a datasource registeredBy", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateRegisteringUser(
+ @RequestParam final String dsId,
+ @RequestParam final String registeredBy) throws DsmException {
+
+ dsmCore.updateDsRegisteringUser(dsId, registeredBy);
+ }
+
+ @Deprecated
+ @RequestMapping(value = "/ds/platform", method = RequestMethod.POST)
+ @ApiOperation(value = "update a datasource platform", notes = "update a datasource platform", tags = { DS, W, D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updatePlatform(
+ @RequestParam final String dsId,
+ @RequestParam final String platform) throws DsmException {
+
+ dsmCore.updateDsPlatform(dsId, platform);
+ }
+
+ @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST)
+ @ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = { API, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateBaseUrl(
+ @RequestParam final String dsId,
+ @RequestParam final String apiId,
+ @RequestParam final String baseUrl) throws DsmException {
+
+ dsmCore.updateApiBaseurl(dsId, apiId, baseUrl);
+ }
+
+ @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST)
+ @ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = { API, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateCompliance(
+ @RequestParam final String dsId,
+ @RequestParam final String apiId,
+ @RequestParam final String compliance,
+ @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException {
+
+ dsmCore.updateApiCompatibility(dsId, apiId, compliance, override);
+ }
+
+ @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST)
+ @ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = { API, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void updateOaiSetl(
+ @RequestParam final String dsId,
+ @RequestParam final String apiId,
+ @RequestParam final String oaiSet) throws DsmException {
+
+ dsmCore.updateApiOaiSet(dsId, apiId, oaiSet);
+ }
+
+ @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST)
+ @ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = { API, W })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void addApi(@RequestBody final ApiDetails api) throws DsmException {
+ if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); }
+ dsmCore.addApi(api);
+ }
+
+ // MANAGEMENT
+
+ @Autowired
+ private OperationManager operationManager;
+
+ @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET)
+ @ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = { R, M })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public int getOps() throws DsmException {
+ return operationManager.getOpSize();
+ }
+
+ @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST)
+ @ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = { W, M })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public int killOps() throws DsmException {
+ return operationManager.dropAll();
+ }
+
+ @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST)
+ @ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = { W, M })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK"),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ public void dropCache() throws DsmException {
+ dsmCore.dropCaches();
+ }
+
+ // OLD DEPRECATED METHODS
+
+ @Deprecated
+ @ApiOperation(value = "search datasources by name", notes = "Returns list of Datasource details.", tags = { D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ @RequestMapping(value = "/ds/search/name/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET)
+ DatasourceSearchResponse searchByName(final String name, final int page, final int size) throws DsmException {
+ final RequestSort sort = RequestSort.id;
+ final RequestSortOrder order = RequestSortOrder.ASCENDING;
+ final RequestFilter filter = new RequestFilter();
+ filter.put(FilterName.englishname, name);
+
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @Deprecated
+ @ApiOperation(value = "search datasources by contact email", notes = "Returns list of Datasource details.", tags = { D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ @RequestMapping(value = "/ds/search/email/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET)
+ DatasourceSearchResponse searchByContactemail(final String contactemail, final int page, final int size) throws DsmException {
+ final RequestSort sort = RequestSort.id;
+ final RequestSortOrder order = RequestSortOrder.ASCENDING;
+ final RequestFilter filter = new RequestFilter();
+ filter.put(FilterName.contactemail, contactemail);
+
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @Deprecated
+ @ApiOperation(value = "search datasources by country", notes = "Returns list of Datasource details.", tags = { D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ @RequestMapping(value = "/ds/search/country/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET)
+ DatasourceSearchResponse searchByCountry(final String country, final Boolean managed, final int page, final int size) throws DsmException {
+ final RequestSort sort = RequestSort.id;
+ final RequestSortOrder order = RequestSortOrder.ASCENDING;
+ final RequestFilter filter = new RequestFilter();
+ filter.put(FilterName.country, country);
+ filter.put(FilterName.managed, managed);
+
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ @Deprecated
+ @ApiOperation(value = "search datasources by registering user", notes = "Returns list of Datasource details.", tags = { D })
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "OK", response = DatasourceSearchResponse.class),
+ @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) })
+ @RequestMapping(value = "/ds/search/registeredby/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET)
+ DatasourceSearchResponse searchByRegisteringUser(final String registeredBy, final int page, final int size) throws DsmException {
+ final RequestSort sort = RequestSort.id;
+ final RequestSortOrder order = RequestSortOrder.ASCENDING;
+ final RequestFilter filter = new RequestFilter();
+ filter.put(FilterName.registeredby, registeredBy);
+
+ final StopWatch stop = StopWatch.createStarted();
+ final DatasourceSearchResponse rsp = dsmCore.search(sort, order, filter, page, size);
+ return prepareResponse(page, size, stop, rsp);
+ }
+
+ // HELPERS
+
+ private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
+ rsp.getHeader()
+ .setTime(stopWatch.getTime())
+ .setPage(page)
+ .setSize(size);
+ return rsp;
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
new file mode 100644
index 00000000..8e9139fd
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
@@ -0,0 +1,579 @@
+package eu.dnetlib.openaire.dsm;
+
+import static eu.dnetlib.openaire.common.ExporterConstants.BASE_URL;
+import static eu.dnetlib.openaire.common.ExporterConstants.COMPLIANCE;
+import static eu.dnetlib.openaire.common.ExporterConstants.ENGLISH_NAME;
+import static eu.dnetlib.openaire.common.ExporterConstants.LATITUDE;
+import static eu.dnetlib.openaire.common.ExporterConstants.LONGITUDE;
+import static eu.dnetlib.openaire.common.ExporterConstants.OAI_SET;
+import static eu.dnetlib.openaire.common.ExporterConstants.OFFICIAL_NAME;
+import static eu.dnetlib.openaire.common.ExporterConstants.PLATFORM;
+import static eu.dnetlib.openaire.common.ExporterConstants.REMOVABLE;
+import static eu.dnetlib.openaire.common.ExporterConstants.TIMEZONE;
+import static eu.dnetlib.openaire.common.ExporterConstants.TYPOLOGY;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDbEntry;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDetails;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asMapOfChanges;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.copyNonNullProperties;
+import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.createId;
+
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.annotation.PostConstruct;
+
+import eu.dnetlib.openaire.dsm.domain.*;
+import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
+import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.jdbc.core.BeanPropertyRowMapper;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.common.xml.XmlEscapers;
+
+import eu.dnetlib.OpenaireExporterConfig;
+import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.enabling.datasources.common.AggregationStage;
+import eu.dnetlib.enabling.datasources.common.Datasource;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
+import eu.dnetlib.openaire.common.ISClient;
+import eu.dnetlib.openaire.community.CommunityClient;
+import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
+import eu.dnetlib.openaire.dsm.dao.DatasourceIndexClient;
+import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
+import eu.dnetlib.openaire.dsm.dao.ObjectStoreClient;
+import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
+import eu.dnetlib.openaire.dsm.dao.VocabularyClient;
+import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo;
+import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
+import eu.dnetlib.openaire.vocabularies.Country;
+
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public class DsmCore {
+
+ private static final Log log = LogFactory.getLog(DsmCore.class);
+
+ @Autowired
+ private MongoLoggerClient mongoLoggerClient;
+
+ @Autowired
+ private ISClient isClient;
+
+ @Autowired
+ private ObjectStoreClient objectStoreClient;
+
+ @Autowired
+ private DatasourceIndexClient datasourceIndexClient;
+
+ @Autowired
+ private VocabularyClient vocabularyClient;
+
+ @Autowired
+ private DatasourceDao dsDao;
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ @Autowired
+ private JdbcTemplate jdbcTemplate;
+
+ @Autowired
+ private CommunityClient communityClient;
+
+ private ListeningExecutorService executor;
+
+ @PostConstruct
+ public void init() {
+ executor = MoreExecutors.listeningDecorator(new ScheduledThreadPoolExecutor(config.getRequestWorkers(),
+ new ThreadFactoryBuilder().setNameFormat("dsm-client-%d").build()));
+ }
+
+ public List listCountries() throws DsmException {
+ try {
+ return dsDao.listCountries();
+ } catch (final Throwable e) {
+ log.error("error listing countries", e);
+ throw e;
+ }
+ }
+
+ @Deprecated
+ public DatasourceSearchResponse search(final RequestSort requestSortBy,
+ final RequestSortOrder order,
+ final RequestFilter requestFilter,
+ final int page,
+ final int size)
+ throws DsmException {
+
+ try {
+ final List datasourceInfo = Lists.newArrayList();
+ final Queue errors = Queues.newLinkedBlockingQueue();
+ final CountDownLatch outerLatch = new CountDownLatch(2);
+
+ final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size);
+ if (dsPage.getTotalElements() > 0 && dsPage.getNumberOfElements() > 0) {
+ dsPage.forEach(d -> datasourceInfo.add(enrichDatasourceInfo(asDetails(d), outerLatch, errors)));
+ waitLatch(outerLatch, errors, config.getRequestTimeout());
+ }
+
+ if (!errors.isEmpty()) {
+ // TODO report on error metrics
+ errors.forEach(log::error);
+ }
+ return ResponseUtils.searchResponse(datasourceInfo, dsPage.getTotalElements());
+ } catch (final Throwable e) {
+ log.error("error searching datasources", e);
+ throw e;
+ }
+ }
+
+ public DatasourceDetailResponse searchDsDetails(final RequestSort requestSortBy,
+ final RequestSortOrder order,
+ final RequestFilter requestFilter,
+ final int page,
+ final int size)
+ throws DsmException {
+
+ try {
+ final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size);
+ return ResponseUtils.detailsResponse(
+ dsPage.map(d -> asDetails(d)).getContent(),
+ dsPage.getTotalElements());
+ } catch (final Throwable e) {
+ log.error("error searching datasources", e);
+ throw e;
+ }
+ }
+
+ public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy,
+ final RequestSortOrder order,
+ final RequestFilter requestFilter,
+ final int page,
+ final int size)
+ throws DsmException {
+ try {
+ final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size);
+ return ResponseUtils.snippetResponse(
+ dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(),
+ dsPage.getTotalElements());
+ } catch (final Throwable e) {
+ log.error("error searching datasources", e);
+ throw e;
+ }
+ }
+
+ public DatasourceSnippetResponse searchRegistered(final RequestSort requestSortBy,
+ final RequestSortOrder order,
+ final RequestFilter requestFilter,
+ final int page,
+ final int size)
+ throws DsmException {
+ try {
+ final Page dsPage = dsDao.searchRegistered(requestSortBy, order, requestFilter, page, size);
+ return ResponseUtils.snippetResponse(
+ dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(),
+ dsPage.getTotalElements());
+ } catch (final Throwable e) {
+ log.error("error searching datasources", e);
+ throw e;
+ }
+ }
+
+ public List findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
+ try {
+ return dsDao.findApiBaseURLs(requestFilter, page, size);
+ } catch (final Throwable e) {
+ log.error("error searching datasource base urls", e);
+ throw e;
+ }
+ }
+
+ public ApiDetailsResponse getApis(final String dsId) throws DsmException {
+ try {
+ final List apis = dsDao.getApis(dsId);
+ final List api = apis.stream()
+ .map(DsmMappingUtils::asDetails)
+ .collect(Collectors.toList());
+ return ResponseUtils.apiResponse(api, api.size());
+ } catch (final Throwable e) {
+ log.error(String.format("error searching datasource api %s", dsId), e);
+ throw e;
+ }
+ }
+
+ public void setManaged(final String dsId, final boolean managed) throws DsmException {
+ log.info(String.format("updated ds '%s' managed with '%s'", dsId, managed));
+ dsDao.setManaged(dsId, managed);
+ final List apis = dsDao.getApis(dsId);
+ for (final ApiDbEntry a : apis) {
+ setApiRemovable(dsId, a.getId(), true);
+ }
+ }
+
+ protected void setApiRemovable(final String dsId, final String apiId, final boolean removable) {
+ log.info(String.format("updated api '%s' removable with '%s'", apiId, removable));
+ final Map changes = Maps.newHashMap();
+ changes.put(REMOVABLE, String.valueOf(removable));
+ isClient.updateAPIField(dsId, apiId, changes);
+ }
+
+ public boolean isManaged(final String dsId) throws DsmException {
+ return dsDao.isManaged(dsId);
+ }
+
+ public boolean exist(final DatasourceDetails d) throws DsmException {
+ return dsDao.existDs(d.getId());
+ }
+
+ public void save(final DatasourceDetails d) throws DsmException {
+ try {
+ dsDao.saveDs(asDbEntry(d));
+ isClient.registerDS(d);
+ } catch (final Throwable e) {
+ log.error(ExceptionUtils.getStackTrace(e));
+ throw e;
+ }
+ }
+
+ public void updateDatasource(final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException {
+ try {
+ // initialize with current values from DB
+ final Datasource ds = dsDao.getDs(d.getId());
+ final DatasourceDbEntry dbEntry = (DatasourceDbEntry) ds;
+
+ if (dbEntry == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); }
+
+ final DatasourceDbEntry update = asDbEntry(d);
+ if (d.getIdentities() != null) {
+ final Set identities = new HashSet<>(
+ Stream.of(update.getIdentities(), dbEntry.getIdentities())
+ .flatMap(Collection::stream)
+ .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1))
+ .values());
+ copyNonNullProperties(update, dbEntry);
+ dbEntry.setIdentities(identities);
+ } else {
+ copyNonNullProperties(update, dbEntry);
+ }
+
+ dsDao.saveDs(dbEntry);
+ isClient.updateDatasourceFields(d.getId(), asMapOfChanges(d));
+ } catch (final Throwable e) {
+ log.error(ExceptionUtils.getStackTrace(e));
+ throw e;
+ }
+ }
+
+ @Deprecated
+ public void updateDatasourcename(final String dsId, final String officialname, final String englishname) throws DsmException {
+ log.info(String.format("updated datasource '%s' with officialname '%s' and englishname '%s'", dsId, officialname, englishname));
+ dsDao.updateName(dsId, officialname, englishname);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(OFFICIAL_NAME, XmlEscapers.xmlContentEscaper().escape(officialname));
+ changes.put(ENGLISH_NAME, XmlEscapers.xmlContentEscaper().escape(englishname));
+ isClient.updateDatasourceFields(dsId, changes);
+ }
+
+ @Deprecated
+ public void updateDatasourceLogoUrl(final String dsId, final String logourl) throws DsmException {
+ log.info(String.format("updated datasource '%s' with logo URL '%s'", dsId, logourl));
+
+ dsDao.updateLogoUrl(dsId, logourl);
+ }
+
+ @Deprecated
+ public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) throws DsmException {
+ log.info(String.format("updated datasource '%s' with coordinates Lat:'%s', Lon:'%s'", dsId, latitude, longitude));
+ dsDao.updateCoordinates(dsId, latitude, longitude);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(LATITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(latitude)));
+ changes.put(LONGITUDE, XmlEscapers.xmlContentEscaper().escape(String.valueOf(longitude)));
+ isClient.updateDatasourceFields(dsId, changes);
+ }
+
+ @Deprecated
+ public void updateTimezone(final String dsId, final String timezone) throws DsmException {
+ log.info(String.format("updated datasource '%s' timezone with '%s'", dsId, timezone));
+ dsDao.updateTimezone(dsId, timezone);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(TIMEZONE, XmlEscapers.xmlContentEscaper().escape(timezone));
+ isClient.updateDatasourceFields(dsId, changes);
+ }
+
+ @Deprecated
+ public void updateDsTypology(final String dsId, final String typology) throws DsmException {
+ log.info(String.format("updated datasource '%s' typology with '%s'", dsId, typology));
+ dsDao.updateTypology(dsId, typology);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(TYPOLOGY, XmlEscapers.xmlContentEscaper().escape(typology));
+ isClient.updateDatasourceFields(dsId, changes);
+ }
+
+ @Deprecated
+ public void updateDsRegisteringUser(final String dsId, final String registeredBy) throws DsmException {
+ log.info(String.format("setting datasource '%s' registering user with '%s'", dsId, registeredBy));
+ dsDao.updateRegisteringUser(dsId, registeredBy);
+ }
+
+ @Deprecated
+ public void updateDsPlatform(final String dsId, final String platform) throws DsmException {
+ log.info(String.format("updated datasource '%s' platform with '%s'", dsId, platform));
+ dsDao.updatePlatform(dsId, platform);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(PLATFORM, XmlEscapers.xmlContentEscaper().escape(platform)); // this is not a typo, Repository profiles map the platform
+ // in the DATASOURCE_TYPE field.
+ isClient.updateDatasourceFields(dsId, changes);
+ }
+
+ // TODO remove if unused
+ public void deleteDs(final String dsId) throws DsmException {
+ log.info(String.format("deleted datasource '%s'", dsId));
+ dsDao.deleteDs(dsId);
+ }
+
+ // API
+
+ public void updateApiOaiSet(final String dsId, final String apiId, final String oaiSet) throws DsmException {
+ final boolean insert = dsDao.upsertApiOaiSet(apiId, oaiSet);
+ final Map changes = Maps.newHashMap();
+ changes.put(OAI_SET, XmlEscapers.xmlContentEscaper().escape(oaiSet));
+
+ if (!insert) {
+ isClient.updateAPIField(dsId, apiId, changes);
+ } else {
+ isClient.addAPIAttribute(dsId, apiId, changes);
+ }
+ }
+
+ public void updateApiBaseurl(final String dsId, final String apiId, final String baseUrl) throws DsmException {
+ log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl));
+ dsDao.updateApiBaseUrl(apiId, baseUrl);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(BASE_URL, XmlEscapers.xmlContentEscaper().escape(baseUrl));
+
+ isClient.updateAPIField(dsId, apiId, changes);
+ }
+
+ public void updateApiCompatibility(final String dsId, final String apiId, final String compliance, final boolean override) throws DsmException {
+ log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance));
+ dsDao.updateCompliance(null, apiId, compliance, override);
+
+ final Map changes = Maps.newHashMap();
+ changes.put(COMPLIANCE, XmlEscapers.xmlAttributeEscaper().escape(compliance));
+
+ isClient.updateAPIField(dsId, apiId, changes);
+ }
+
+ public void addApi(final ApiDetails api) throws DsmException {
+ if (StringUtils.isBlank(api.getId())) {
+ api.setId(createId(api));
+ log.info(String.format("missing api id, created '%s'", api.getId()));
+ }
+
+ dsDao.addApi(asDbEntry(api));
+ isClient.registerAPI(api);
+ }
+
+ public void deleteApi(final String apiId) throws DsmForbiddenException, DsmNotFoundException {
+ // TODO handle the api removal in case of associated workflows.
+ isClient.removeAPI(apiId);
+ dsDao.deleteApi(null, apiId);
+ }
+
+ public void dropCaches() {
+ mongoLoggerClient.dropCache();
+ isClient.dropCache();
+ vocabularyClient.dropCache();
+ communityClient.dropCache();
+ }
+
+ // HELPERS //////////////
+
+ private DatasourceInfo enrichDatasourceInfo(final DatasourceDetails d, final CountDownLatch outerLatch, final Queue errors) {
+ final DatasourceInfo dsInfo = new DatasourceInfo().setDatasource(d);
+ getAggregationHistory(d.getId(), outerLatch, errors, dsInfo);
+ getIndexDsInfo(d.getId(), outerLatch, errors, dsInfo);
+ return dsInfo;
+ }
+
+ private void getAggregationHistory(final String dsId,
+ final CountDownLatch outerLatch,
+ final Queue errors,
+ final DatasourceInfo datasourceInfo) {
+ Futures.addCallback(executor.submit(() -> mongoLoggerClient.getAggregationHistory(dsId)), new FutureCallback>() {
+
+ @Override
+ public void onSuccess(final List info) {
+ setAggregationHistory(datasourceInfo, info);
+ outerLatch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ log.error(ExceptionUtils.getStackTrace(e));
+ errors.offer(e);
+ outerLatch.countDown();
+ }
+ }, executor);
+ }
+
+ private void setAggregationHistory(final DatasourceInfo datasourceInfo, final List info) {
+ datasourceInfo.setAggregationHistory(info);
+ if (!info.isEmpty()) {
+ datasourceInfo
+ .setLastCollection(info.stream().filter(a -> AggregationStage.COLLECT.equals(a.getAggregationStage())).findFirst().get())
+ .setLastTransformation(info.stream().filter(a -> AggregationStage.TRANSFORM.equals(a.getAggregationStage())).findFirst().get());
+ }
+ }
+
+ private void getIndexDsInfo(final String dsId,
+ final CountDownLatch outerLatch,
+ final Queue errors,
+ final DatasourceInfo datasourceInfo) {
+ Futures.addCallback(executor.submit(() -> isClient.calculateCurrentIndexDsInfo()), new FutureCallback() {
+
+ @Override
+ public void onSuccess(final IndexDsInfo info) {
+
+ final CountDownLatch innerLatch = new CountDownLatch(2);
+
+ Futures.addCallback(executor.submit(() -> datasourceIndexClient.getIndexInfo(dsId, info, errors)), new FutureCallback() {
+
+ @Override
+ public void onSuccess(final IndexRecordsInfo info) {
+ datasourceInfo
+ .setIndexRecords(info.getTotal())
+ .setFundedContent(info.getFunded())
+ .setLastIndexingDate(info.getDate());
+ innerLatch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ errors.offer(e);
+ innerLatch.countDown();
+ }
+ }, executor);
+
+ Futures.addCallback(executor.submit(() -> objectStoreClient.getObjectStoreSize(isClient.getObjectStoreId(dsId))), new FutureCallback() {
+
+ @Override
+ public void onSuccess(final Long objectStoreSize) {
+ datasourceInfo.setFulltexts(objectStoreSize);
+ innerLatch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ errors.offer(e);
+ innerLatch.countDown();
+ }
+ }, executor);
+
+ waitLatch(innerLatch, errors, config.getRequestTimeout());
+
+ outerLatch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ // log.error(ExceptionUtils.getStackTrace(e));
+ errors.offer(e);
+ outerLatch.countDown();
+ }
+ }, executor);
+ }
+
+ private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) {
+ try {
+ if (!latch.await(waitSeconds, TimeUnit.SECONDS)) {
+ errors.offer(new TimeoutException("Waiting for requests to complete has timed out."));
+ }
+ } catch (final InterruptedException e) {
+ errors.offer(e);
+ }
+ }
+
+ public SimpleResponse searchRecentRegistered(final int size) throws Throwable {
+ try {
+ final String sql =
+ IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset());
+
+ final List list = jdbcTemplate.query(sql, BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size);
+
+ return ResponseUtils.simpleResponse(list);
+ } catch (final Throwable e) {
+ log.error("error searching recent datasources", e);
+ throw e;
+ }
+ }
+ public Long countRegisteredAfter(final String fromDate, final String typologyFilter) throws Throwable {
+ try {
+ if (StringUtils.isNotBlank(typologyFilter)) {
+ final String sql =
+ IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset.defaultCharset());
+
+ return jdbcTemplate.queryForObject(sql, new Object[] { fromDate, typologyFilter + "%" }, Long.class);
+ } else {
+ final String sql =
+ IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset.defaultCharset());
+
+ return jdbcTemplate.queryForObject(sql, new Object[] { fromDate }, Long.class);
+ }
+
+
+ } catch (final Throwable e) {
+ log.error("error searching recent datasources", e);
+ throw e;
+ }
+ }
+
+ public AggregationHistoryResponse aggregationhistory(String dsId) throws DsmException {
+ final List history = mongoLoggerClient.getAggregationHistory(dsId);
+ final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history);
+ rsp.setHeader(ResponseUtils.header(history.size()));
+ return rsp;
+
+ }
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java
new file mode 100644
index 00000000..044ed440
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java
@@ -0,0 +1,53 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import java.util.List;
+import javax.transaction.Transactional;
+
+import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+
+/**
+ * Created by claudio on 15/06/2017.
+ */
+@Repository
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public interface ApiDbEntryRepository extends JpaRepository {
+
+ @Query("select a from #{#entityName} a where a.datasource = ?1")
+ List findByDatasource(String dsId);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} a set a.baseurl = ?2 where a.id = ?1")
+ void setBaseurl(String id, String baseurl);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} a set a.compatibility = ?2 where a.id = ?1")
+ void updateCompatibility(String apiId, String compatibility);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} a set a.compatibilityOverride = ?2 where a.id = ?1")
+ void updateCompatibilityOverride(String apiId, String compatibility);
+
+ @Modifying
+ @Transactional
+ @Query(value = "update dsm_apiparams ap set value = ?2 where ap.param = 'set' and ap.api = ?1", nativeQuery = true)
+ void updateOaiSet(String apiId, String oaiSet);
+
+ @Modifying
+ @Transactional
+ @Query(value = "insert into dsm_apiparams(api, param, value, _dnet_resource_identifier_) values(?1, ?2, ?3, ?1||'@@'||?2)", nativeQuery = true)
+ void addApiParam(String apiId, String param, String value);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.removable = ?2 where d.datasource = ?1")
+ void setRemovable(String id, boolean removable);
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java
new file mode 100644
index 00000000..221810f1
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java
@@ -0,0 +1,15 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import eu.dnetlib.openaire.dsm.domain.db.CountryTerm;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+/**
+ * Created by claudio on 19/04/2017.
+ */
+@Repository
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public interface CountryTermRepository extends JpaRepository {
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java
new file mode 100644
index 00000000..af13e141
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java
@@ -0,0 +1,14 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.stereotype.Repository;
+
+@Repository
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public interface DatasourceApiDbEntryRepository extends JpaRepository, JpaSpecificationExecutor {
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java
new file mode 100644
index 00000000..4fb71855
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java
@@ -0,0 +1,58 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import java.util.List;
+
+import eu.dnetlib.enabling.datasources.common.Api;
+import eu.dnetlib.enabling.datasources.common.Datasource;
+import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.openaire.dsm.domain.RequestFilter;
+import eu.dnetlib.openaire.dsm.domain.RequestSort;
+import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
+import eu.dnetlib.openaire.vocabularies.Country;
+import org.springframework.data.domain.Page;
+
+public interface DatasourceDao, API extends Api>> extends DatasourceManagerCommon {
+
+ // DATASOURCE
+
+ List listCountries() throws DsmException;
+
+ boolean existDs(final String dsId) throws DsmException;
+
+ Page search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException;
+
+ Page searchRegistered(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException;
+
+ void updateName(String dsId, String officialname, String englishname) throws DsmException;
+
+ void updateLogoUrl(String dsId, String logourl) throws DsmException;
+
+ void updateCoordinates(String dsId, Double latitude, Double longitude) throws DsmException;
+
+ void updateTimezone(String dsId, String timezone) throws DsmException;
+
+ void updateTypology(String dsId, String timezone) throws DsmException;
+
+ void updateRegisteringUser(String dsId, String registeredBy) throws DsmException;
+
+ void updatePlatform(String dsId, String platform) throws DsmException;
+
+ // API
+
+ List findApiBaseURLs(RequestFilter requestFilter, int page, int size) throws DsmException;
+
+ /**
+ * Insert the oai set in case it does not exists, updates it otherwise
+ *
+ * @param apiId
+ * @param oaiSet
+ * @return true in case of insert, false in case of update
+ * @throws DsmException
+ */
+ boolean upsertApiOaiSet(String apiId, String oaiSet) throws DsmException;
+
+ void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException;
+
+ void addApi(final API api) throws DsmException;
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java
new file mode 100644
index 00000000..cb53f505
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java
@@ -0,0 +1,262 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.OpenaireExporterConfig;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
+import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
+import eu.dnetlib.openaire.dsm.domain.RequestFilter;
+import eu.dnetlib.openaire.dsm.domain.RequestSort;
+import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
+import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
+import eu.dnetlib.openaire.vocabularies.Country;
+import eu.dnetlib.openaire.vocabularies.Vocabulary;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpStatus;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.jpa.domain.Specification;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.persistence.EntityNotFoundException;
+import java.sql.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static eu.dnetlib.openaire.common.ExporterConstants.OAI;
+import static eu.dnetlib.openaire.common.ExporterConstants.SET;
+import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec;
+import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec;
+import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec;
+
+
+/**
+ * Created by claudio on 20/10/2016.
+ */
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public class DatasourceDaoImpl implements DatasourceDao {
+
+ private static final Log log = LogFactory.getLog(DatasourceDao.class);
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ @Autowired
+ private CountryTermRepository countryTermRepository;
+
+ @Autowired
+ private DatasourceDbEntryRepository dsRepository;
+
+ @Autowired
+ private ApiDbEntryRepository apiRepository;
+
+ @Autowired
+ private DatasourceApiDbEntryRepository dsApiRepository;
+
+ @Autowired
+ private VocabularyClient vocabularyClient;
+
+ @Override
+ public List listCountries() throws DsmException {
+ final List countries = Lists.newArrayList();
+ final Vocabulary v = vocabularyClient.getCountries();
+ countries.addAll(countryTermRepository.findAll().stream()
+ .filter(Objects::nonNull)
+ .map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm())))
+ .collect(Collectors.toList()));
+ return countries;
+ }
+
+ @Override
+ public Page search(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size)
+ throws DsmException {
+
+ final Specification spec = dsSpec(requestSortBy, order, requestFilter);
+ return dsRepository.findAll(spec, PageRequest.of(page, size));
+ }
+
+
+ @Override
+ public Page searchRegistered(final RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, final int page, final int size)
+ throws DsmException {
+
+ final Specification spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec());
+ return dsRepository.findAll(spec, PageRequest.of(page, size));
+ }
+
+ @Override
+ public DatasourceDbEntry getDs(final String dsId) throws DsmException {
+ return dsRepository.getOne(dsId);
+ }
+
+ @Override
+ public void setManaged(final String id, final boolean managed) {
+ log.info(String.format("setting managed = '%s' for ds '%s'", managed, id));
+ dsRepository.setManaged(id, managed);
+ apiRepository.setRemovable(id, true);
+ }
+
+ @Override
+ public boolean isManaged(final String id) {
+ return dsRepository.isManaged(id);
+ }
+
+ @Override
+ public void updateCompliance(String dsId, String apiId, String compliance, boolean override) {
+ log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId));
+ apiRepository.updateCompatibility(apiId, compliance);
+ }
+
+ @Override
+ public List getApis(final String dsId) {
+ return apiRepository.findByDatasource(dsId);
+ }
+
+ @Override
+ public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException {
+ final ApiDbEntry api = apiRepository.getOne(apiId);
+ try {
+ if (!api.getRemovable()) {
+ throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable");
+ }
+
+ apiRepository.deleteById(apiId);
+ log.info(String.format("deleted api '%s'", apiId));
+ } catch (EntityNotFoundException e) {
+ throw new DsmNotFoundException(HttpStatus.SC_NOT_FOUND, "api not found");
+ }
+ }
+
+ @Override
+ public void addApi(final ApiDbEntry api) {
+ apiRepository.save(api);
+ }
+
+ @Override
+ public boolean existDs(final String dsId) throws DsmException {
+ return dsRepository.existsById(dsId);
+ }
+
+ @Override
+ public void saveDs(final DatasourceDbEntry d) {
+ log.info(String.format("saving datasource '%s'", d.getId()));
+
+ final DatasourceDbEntry datasource = dsRepository.save(d);
+ log.info(String.format("saved datasource '%s'", datasource.getId()));
+
+ ensureRegistrationDate(d.getId());
+ }
+
+ @Override
+ public void deleteDs(final String dsId) {
+ dsRepository.deleteById(dsId);
+ log.info(String.format("deleted datasource '%s'", dsId));
+ }
+
+ @Override
+ public void updateName(final String dsId, final String officialname, final String englishname) {
+ //TODO what if one of the two names is null or empty?
+ dsRepository.setDatasourcename(dsId, officialname, englishname);
+ }
+
+ @Override
+ public void updateLogoUrl(final String dsId, final String logourl) throws DsmException {
+ dsRepository.setLogoUrl(dsId, logourl);
+ }
+
+ @Override
+ public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) {
+ dsRepository.setCoordinates(dsId, latitude, longitude);
+ }
+
+ @Override
+ public void updateApiBaseUrl(final String apiId, final String baseurl) {
+ apiRepository.setBaseurl(apiId, baseurl);
+ }
+
+ @Override
+ @Transactional
+ public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException {
+ final ApiDbEntry api = apiRepository.getOne(apiId);
+ if (OAI.equalsIgnoreCase(api.getProtocol())) {
+ final Set apiParams = api.getApiParams();
+
+ if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) {
+ apiRepository.addApiParam(apiId, SET, oaiSet);
+ log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet));
+ return true;
+ } else {
+ apiRepository.updateOaiSet(apiId, oaiSet);
+ log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet));
+ return false;
+ }
+ } else {
+ throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol()));
+ }
+ }
+
+ @Override
+ public List findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
+ final PageRequest pageable = PageRequest.of(page, size);
+ final Specification spec = apiSpec(requestFilter);
+ final Set set = dsApiRepository.findAll(spec, pageable).getContent().stream()
+ .map(DatasourceApiDbEntry::getBaseurl)
+ .filter(StringUtils::isNotBlank)
+ .collect(Collectors.toCollection(HashSet::new));
+ return Lists.newArrayList(set);
+ }
+
+ @Override
+ public void updateTimezone(final String dsId, final String timezone) {
+ dsRepository.setTimezone(dsId, timezone);
+ }
+
+ @Override
+ public void updateTypology(final String dsId, final String typology) throws DsmException {
+ final Vocabulary typologies = vocabularyClient.getDatasourceTypologies();
+ if (!typologies.hasCode(typology)) {
+ throw new DsmException(
+ HttpStatus.SC_BAD_REQUEST,
+ String.format(
+ "invalid datasource typology '%s', provide one according to vocabulary %s",
+ typology,
+ config.getVocabularies().getDatasourceTypologiesEndpoint()));
+ }
+ dsRepository.setTypology(dsId, typology);
+ }
+
+ @Override
+ public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException {
+
+ ensureRegistrationDate(dsId);
+
+ dsRepository.setRegisteringUser(dsId, registeredBy);
+
+ }
+
+ @Override
+ public void updatePlatform(final String dsId, final String platform) throws DsmException {
+ dsRepository.setPlatform(dsId, platform);
+ }
+
+ //HELPER
+ private void ensureRegistrationDate(String dsId) {
+ if (!dsRepository.hasRegistrationdate(dsId)) {
+ log.info("setting registration date for datasource: " + dsId);
+ dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis()));
+ }
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java
new file mode 100644
index 00000000..5771b7d4
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java
@@ -0,0 +1,75 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import javax.transaction.Transactional;
+
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+
+import java.sql.Date;
+
+/**
+ * Created by claudio on 12/04/2017.
+ */
+@Repository
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public interface DatasourceDbEntryRepository extends JpaRepository, JpaSpecificationExecutor {
+
+ DatasourceDbEntry findOneById(String id);
+
+ @Query("select d.managed from #{#entityName} d where d.id = ?1")
+ boolean isManaged(String id);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.managed = ?2 where d.id = ?1")
+ void setManaged(String id, boolean managed);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.officialname = ?2, d.englishname = ?3 where d.id = ?1")
+ void setDatasourcename(String id, String officialname, String englishname);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.logourl = ?2 where d.id = ?1")
+ void setLogoUrl(String dsId, String logourl);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.latitude = ?2, d.longitude = ?3 where d.id = ?1")
+ void setCoordinates(String dsId, Double latitude, Double longitude);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.timezone = ?2 where d.id = ?1")
+ void setTimezone(String dsId, String timezone);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.typology = ?2 where d.id = ?1")
+ void setTypology(String dsId, String typology);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.registeredby = ?2 where d.id = ?1")
+ void setRegisteringUser(String id, String registeredby);
+
+ @Query("select case when registrationdate <> null then true else false end as hasregistrationdate from #{#entityName} where id = ?1")
+ Boolean hasRegistrationdate(String id);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1")
+ void setRegistrationDate(String id, Date registrationdate);
+
+ @Modifying
+ @Transactional
+ @Query("update #{#entityName} d set d.platform = ?2 where d.id = ?1")
+ void setPlatform(String id, String platform);
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java
new file mode 100644
index 00000000..79d8e5e8
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java
@@ -0,0 +1,17 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import java.util.Queue;
+
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo;
+
+public interface DatasourceIndexClient {
+
+ IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException;
+
+ String getLastIndexingDate(final IndexDsInfo info) throws DsmException;
+
+
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java
new file mode 100644
index 00000000..0788bed1
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java
@@ -0,0 +1,207 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.*;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.*;
+import eu.dnetlib.OpenaireExporterConfig;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
+import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+
+/**
+ * Created by claudio on 20/10/2016.
+ */
+@Component
+@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
+public class DatasourceIndexClientImpl implements DatasourceIndexClient {
+
+ private static final Log log = LogFactory.getLog(DatasourceIndexClientImpl.class);
+
+ public static final String SEPARATOR = "::";
+ public static final String DSVERSION = "__dsversion";
+
+ @Autowired
+ private OpenaireExporterConfig config;
+
+ private ListeningExecutorService executor;
+
+ private static final Map indexClientMap = new ConcurrentHashMap<>();
+
+ @PostConstruct
+ public void init() {
+ executor = MoreExecutors.listeningDecorator(
+ new ScheduledThreadPoolExecutor(5,
+ new ThreadFactoryBuilder().setNameFormat("datasource-index-client-%d").build()));
+ }
+
+ @PreDestroy
+ public void tearDown() {
+ indexClientMap.forEach((name, client) -> {
+ try {
+ client.close();
+ } catch (IOException e) {
+ log.warn(String.format("unable to gracefully shutdown client for index %s", name));
+ }
+ });
+ }
+
+ @Override
+ public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException {
+ try {
+ final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR));
+ final CloudSolrClient indexClient = getIndexClient(info);
+ final CountDownLatch latch = new CountDownLatch(2);
+ final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo();
+
+ Futures.addCallback(
+ executor.submit(() -> setDateAndTotal(dsId, collectedFrom, indexClient)),
+ new FutureCallback() {
+
+ @Override
+ public void onSuccess(final IndexRecordsInfo info) {
+ indexRecordInfo
+ .setTotal(info.getTotal())
+ .setDate(info.getDate());
+ latch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ errors.offer(e);
+ latch.countDown();
+ }
+ }, executor);
+
+ Futures.addCallback(
+ executor.submit(() -> setFunded(dsId, collectedFrom, indexClient)),
+ new FutureCallback() {
+
+ @Override
+ public void onSuccess(final Long numFound) {
+ indexRecordInfo.setFunded(numFound);
+ latch.countDown();
+ }
+
+ @Override
+ public void onFailure(final Throwable e) {
+ errors.offer(e);
+ latch.countDown();
+ }
+ }, executor);
+
+ waitLatch(latch, errors, config.getRequestTimeout());
+ return indexRecordInfo;
+ } catch (final Throwable e) {
+ throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("error reading index info", dsId), e);
+ }
+ }
+
+ @Override
+ public String getLastIndexingDate(final IndexDsInfo info) throws DsmException {
+ try {
+ final SolrQuery query = new SolrQuery("oaftype:datasource").setRows(1);
+ final QueryResponse rsp = getIndexClient(info).query(query);
+ final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);
+ final String dsversion = doc.get("__dsversion").toString();
+ return StringUtils.substringBefore(dsversion, "T");
+ } catch (SolrServerException | IOException e) {
+ throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e);
+ }
+ }
+
+ private Long setFunded(
+ final String dsId,
+ final String collectedFrom,
+ final CloudSolrClient indexClient) throws DsmException {
+ final String query =
+ String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom);
+ log.debug(String.format("query: %s", query));
+ try {
+ return indexClient.query(new SolrQuery(query).setRows(0)).getResults().getNumFound();
+ } catch (Throwable e) {
+ throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for funded results '%s'", dsId), e);
+ }
+ }
+
+ private IndexRecordsInfo setDateAndTotal(
+ final String dsId,
+ final String collectedFrom,
+ final CloudSolrClient indexClient) throws DsmException {
+ try {
+ final String query = String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom);
+ log.debug(String.format("query: %s", query));
+
+ final QueryResponse rsp = indexClient.query(new SolrQuery(query).setRows(1));
+ final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument());
+ if (log.isDebugEnabled()) {
+ log.debug(String.format("got document %s", doc.get("__indexrecordidentifier")));
+ }
+ // if (doc.isEmpty()) {
+ // throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching
+ // query: %s", queryTotal));
+ // }
+ return new IndexRecordsInfo()
+ .setDate(getDate(doc))
+ .setTotal(rsp.getResults().getNumFound());
+ } catch (Throwable e) {
+ throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for date and total '%s'", dsId), e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private String getDate(final SolrDocument doc) throws DsmException {
+ final List dsversion = (List) doc.get(DSVERSION);
+ if (dsversion == null || dsversion.isEmpty()) { throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(),
+ String.format("cannot find %s in matched solr document", DSVERSION)); }
+ final Date date = Iterables.getLast(dsversion);
+
+ return DateFormatUtils.format(date, DsmMappingUtils.DATE_FORMAT);
+ }
+
+ private synchronized CloudSolrClient getIndexClient(final IndexDsInfo info) {
+ if (!indexClientMap.containsKey(info.getColl())) {
+
+ final CloudSolrClient client = new Builder(Lists.newArrayList(info.getIndexBaseUrl())).build();
+ client.setDefaultCollection(info.getColl());
+
+ indexClientMap.put(info.getColl(), client);
+ }
+ return indexClientMap.get(info.getColl());
+ }
+
+ private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) {
+ try {
+ if (!latch.await(waitSeconds, TimeUnit.SECONDS)) {
+ errors.offer(new TimeoutException("Waiting for requests to complete has timed out."));
+ }
+ } catch (final InterruptedException e) {
+ errors.offer(e);
+ }
+ }
+
+}
diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java
new file mode 100644
index 00000000..218c527d
--- /dev/null
+++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java
@@ -0,0 +1,141 @@
+package eu.dnetlib.openaire.dsm.dao;
+
+import eu.dnetlib.enabling.datasources.common.DsmRuntimeException;
+import eu.dnetlib.openaire.dsm.domain.FilterName;
+import eu.dnetlib.openaire.dsm.domain.RequestFilter;
+import eu.dnetlib.openaire.dsm.domain.RequestSort;
+import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
+import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.data.jpa.domain.Specification;
+
+import javax.persistence.criteria.*;
+import java.util.List;
+import java.util.Map.Entry;
+
+public class DatasourceSpecs {
+
+ private static final Log log = LogFactory.getLog(DatasourceSpecs.class);
+
+ public static final String WILDCARD = "%";
+
+ public static Specification dsRegisteredbyNotNullSpec() {
+ return (ds, query, cb) -> cb.and(
+ cb.isNull(ds.get(FilterName.registeredby.name())).not(),
+ cb.isNull(ds.get("registrationdate")).not());
+ }
+
+ public static Specification dsSpec(final RequestSort requestSortBy, final RequestSortOrder order, final RequestFilter requestFilter) {
+ log.debug(String.format("RequestFilter:'%s', RequestSort:'%s', RequestSortOrder:'%s'", requestFilter, requestSortBy, order));
+ return (ds, query, cb) -> {
+ final Predicate p = cb.conjunction();
+ if (requestFilter != null) {
+ final List> expressions = p.getExpressions();
+ requestFilter.entrySet().stream()
+ .forEach(e -> {
+ switch (FilterName.type(e.getKey())) {
+
+ case exact:
+ expressions.add(exactSearch(ds, cb, e));
+
+ break;
+ case search:
+ expressions.add(likeSearch(ds, cb, e));
+
+ break;
+ case searchOrgs:
+ // search by case insensitive organization's country
+ expressions.add(
+ cb.equal(
+ cb.lower(
+ ds.join("organizations").get(FilterName.country.name())),
+ getValue(e)));
+ break;
+ }
+ });
+ }
+ if (requestSortBy != null) {
+ if (order != null) {
+ final Path