D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'new-exporter' (#5) from new-exporter into master 

Reviewed-on: #5
This commit is contained in:
Michele Artini 2022-04-22 10:27:22 +02:00
parent b5b130e0ab a15ececddd
commit 3e405075c9
176 changed files with 14543 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/dnet-exporter-api/README.md Normal file
View File

@ -0,0 +1,6 @@
SpringBoot application implementing OpenAIRE REST API to manage
- Datasources
- Contexts
- Communities
- Funders
- Projects

149
apps/dnet-exporter-api/pom.xml Normal file
View File

@ -0,0 +1,149 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dnet-exporter-api</artifactId>
<packaging>jar</packaging>
<!-- Add typical dependencies for a web application -->
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>cnr-rmi-api</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-rt-transports-http</artifactId>
<version>3.1.5</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>cnr-service-common</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-openaireplus-mapping-utils</artifactId>
<version>[6.3.0,7.0.0)</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-hadoop-commons</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-objectstore-rmi</artifactId>
<version>[2.0.0,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>7.5.0</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</dependency>
<dependency>
<groupId>net.sf.supercsv</groupId>
<artifactId>super-csv</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-joda</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>dnet-datasource-manager-common</artifactId>
<version>[2.0.1,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib;
import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer;
import org.springframework.cache.concurrent.ConcurrentMapCacheManager;
import org.springframework.stereotype.Component;
import static java.util.Arrays.asList;
@Component
public class CacheCustomizer implements CacheManagerCustomizer<ConcurrentMapCacheManager> {
@Override
public void customize(final ConcurrentMapCacheManager cacheManager) {
cacheManager.setCacheNames(
asList(
"fundingpath-ids",
"indexdsinfo-cache",
"objectstoreid-cache",
"context-cache",
"context-cache-funder",
"context-cache-community",
"dsm-aggregationhistory-cache",
"dsm-firstharvestdate-cache",
"vocabularies-cache",
"community-cache",
"info"));
}
}

110
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java Normal file
View File

@ -0,0 +1,110 @@
package eu.dnetlib;
import static springfox.documentation.builders.RequestHandlerSelectors.basePackage;
import java.time.LocalDate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.DnetOpenaireExporterProperties.Swagger;
import eu.dnetlib.common.app.AbstractDnetApp;
import eu.dnetlib.openaire.community.CommunityApiController;
import eu.dnetlib.openaire.context.ContextApiController;
import eu.dnetlib.openaire.dsm.DsmApiController;
import eu.dnetlib.openaire.funders.FundersApiController;
import eu.dnetlib.openaire.info.InfoController;
import eu.dnetlib.openaire.project.ProjectsController;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.Contact;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@EnableCaching
@EnableScheduling
@EnableSwagger2
@SpringBootApplication
@EnableAutoConfiguration(exclude = {
SolrAutoConfiguration.class
})
public class DNetOpenaireExporterApplication extends AbstractDnetApp {
public static final String V1 = "1.0.0";
public static void main(final String[] args) throws Exception {
SpringApplication.run(DNetOpenaireExporterApplication.class, args);
}
@Autowired
private DnetOpenaireExporterProperties config;
@Bean
public Docket dsm() {
return _docket("Datasource Manager", DsmApiController.class.getPackage().getName(), config.getSwaggerDsm(), V1);
}
@Bean
public Docket projects() {
return _docket("OpenAIRE Projects", ProjectsController.class.getPackage().getName(), config.getSwaggerProjects(), V1);
}
@Bean
public Docket funders() {
return _docket("OpenAIRE Funders", FundersApiController.class.getPackage().getName(), config.getSwaggerFunders(), V1);
}
@Bean
public Docket communities() {
return _docket("OpenAIRE Communities", CommunityApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1);
}
@Bean
public Docket contexts() {
return _docket("OpenAIRE Contexts", ContextApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1);
}
private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) {
final Docket d = new Docket(DocumentationType.SWAGGER_2);
configSwagger(d, groupName, controllerPackage, swag, version);
return d;
}
@Override
protected void configSwagger(final Docket docket) {
configSwagger(docket, "OpenAIRE Info", InfoController.class.getPackage().getName(), config.getSwaggerInfo(), V1);
}
private void configSwagger(final Docket docket, final String groupName, final String controllerPackage, final Swagger swag, final String version) {
docket
.groupName(groupName)
.select()
.apis(basePackage(controllerPackage))
.build()
.directModelSubstitute(LocalDate.class, java.sql.Date.class)
.apiInfo(apiInfo(swag, version));
}
private ApiInfo apiInfo(final Swagger swag, final String version) {
return new ApiInfoBuilder()
.title(swag.getApiTitle())
.description(swag.getApiDescription())
.license(swag.getApiLicense())
.licenseUrl(swag.getApiLicenseUrl())
.termsOfServiceUrl("")
.version(version)
.contact(new Contact(
swag.getApiContactName(),
swag.getApiContactUrl(),
swag.getApiContactEmail()))
.build();
}
}

106
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java Normal file
View File

@ -0,0 +1,106 @@
package eu.dnetlib;
import javax.sql.DataSource;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.cxf.endpoint.Client;
import org.apache.cxf.frontend.ClientProxy;
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
import org.apache.cxf.transport.http.HTTPConduit;
import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import eu.dnetlib.DnetOpenaireExporterProperties.Jdbc;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
/**
* Created by claudio on 07/07/2017.
*/
@Configuration
public class DNetOpenaireExporterConfiguration {
private static final Log log = LogFactory.getLog(DNetOpenaireExporterConfiguration.class);
@Autowired
private DnetOpenaireExporterProperties props;
@Bean
public ISLookUpService getLookUpService() {
return getServiceStub(ISLookUpService.class, props.getIsLookupUrl());
}
@Bean
public ObjectStoreService getObjectStoreService() {
return getServiceStub(ObjectStoreService.class, props.getObjectStoreServiceUrl());
}
@Bean
public ISRegistryService getRegistryService() {
return getServiceStub(ISRegistryService.class, props.getIsRegistryServiceUrl());
}
@SuppressWarnings("unchecked")
private <T> T getServiceStub(final Class<T> clazz, final String endpoint) {
log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(), endpoint));
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
jaxWsProxyFactory.setServiceClass(clazz);
jaxWsProxyFactory.setAddress(endpoint);
final T service = (T) jaxWsProxyFactory.create();
final Client client = ClientProxy.getClient(service);
if (client != null) {
final HTTPConduit conduit = (HTTPConduit) client.getConduit();
final HTTPClientPolicy policy = new HTTPClientPolicy();
log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", props.getCxfClientConnectTimeout(), props
.getCxfClientReceiveTimeout(), clazz.getCanonicalName()));
policy.setConnectionTimeout(props.getCxfClientConnectTimeout());
policy.setReceiveTimeout(props.getCxfClientReceiveTimeout());
conduit.setClient(policy);
}
return service;
}
@Bean
public DataSource getSqlDataSource() {
final Jdbc jdbc = props.getJdbc();
return getDatasource(jdbc.getDriverClassName(), jdbc.getUrl(), jdbc.getUser(), jdbc.getPwd(), jdbc.getMinIdle(), jdbc.getMaxRows());
}
private BasicDataSource getDatasource(final String driverClassName,
final String jdbcUrl,
final String jdbcUser,
final String jdbcPwd,
final int jdbcMinIdle,
final int jdbcMaxIdle) {
final BasicDataSource d = new BasicDataSource();
d.setDriverClassName(driverClassName);
d.setUrl(jdbcUrl);
d.setUsername(jdbcUser);
d.setPassword(jdbcPwd);
d.setMinIdle(jdbcMinIdle);
d.setMaxIdle(jdbcMaxIdle);
return d;
}
@Bean
public MongoClient getMongoClient() {
return new MongoClient(
new ServerAddress(props.getDatasource().getMongoHost(), props.getDatasource().getMongoPort()),
MongoClientOptions.builder().connectionsPerHost(props.getDatasource().getMongoConnectionsPerHost()).build());
}
}

548
apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java Normal file
View File

@ -0,0 +1,548 @@
package eu.dnetlib;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
/**
* Created by Alessia Bardi on 31/03/17.
*
* @author Alessia Bardi, Claudio Atzori
*/
@Configuration
@PropertySource("classpath:global.properties")
@ConfigurationProperties(prefix = "openaire.exporter")
public class DnetOpenaireExporterProperties {
// ISLOOKUP
private ClassPathResource findSolrIndexUrl;
private ClassPathResource findIndexDsInfo;
private ClassPathResource findObjectStore;
private ClassPathResource findFunderContexts;
private ClassPathResource findCommunityContexts;
private ClassPathResource findContextProfiles;
private ClassPathResource findContextProfilesByType;
private ClassPathResource getRepoProfile;
@Value("${openaire.exporter.contentLoadQuery}")
private String contentLoadQuery;
private String isLookupUrl;
private String objectStoreServiceUrl;
private String isRegistryServiceUrl;
private int requestWorkers = 100;
private int requestTimeout = 10;
private int cxfClientConnectTimeout = 120;
private int cxfClientReceiveTimeout = 120;
private Datasource datasource;
private Project project;
private Jdbc jdbc;
private Swagger swaggerDsm;
private Swagger swaggerProjects;
private Swagger swaggerFunders;
private Swagger swaggerCommunities;
private Swagger swaggerContexts;
private Swagger swaggerInfo;
private Vocabularies vocabularies;
public static class Datasource {
// MONGODB
private String mongoHost;
private int mongoPort;
private String mongoCollectionName;
private String mongoDbName;
private int mongoConnectionsPerHost;
private int mongoQueryLimit;
public String getMongoHost() {
return mongoHost;
}
public void setMongoHost(final String mongoHost) {
this.mongoHost = mongoHost;
}
public int getMongoPort() {
return mongoPort;
}
public void setMongoPort(final int mongoPort) {
this.mongoPort = mongoPort;
}
public String getMongoCollectionName() {
return mongoCollectionName;
}
public void setMongoCollectionName(final String mongoCollectionName) {
this.mongoCollectionName = mongoCollectionName;
}
public String getMongoDbName() {
return mongoDbName;
}
public void setMongoDbName(final String mongoDbName) {
this.mongoDbName = mongoDbName;
}
public int getMongoConnectionsPerHost() {
return mongoConnectionsPerHost;
}
public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) {
this.mongoConnectionsPerHost = mongoConnectionsPerHost;
}
public int getMongoQueryLimit() {
return mongoQueryLimit;
}
public void setMongoQueryLimit(final int mongoQueryLimit) {
this.mongoQueryLimit = mongoQueryLimit;
}
}
public static class Project {
private int flushSize;
private String tsvFields;
private Resource projectsFundingQueryTemplate;
private Resource dspaceTemplate;
private Resource dspaceHeadTemplate;
private Resource dspaceTailTemplate;
private Resource eprintsTemplate;
public int getFlushSize() {
return flushSize;
}
public void setFlushSize(final int flushSize) {
this.flushSize = flushSize;
}
public String getTsvFields() {
return tsvFields;
}
public void setTsvFields(final String tsvFields) {
this.tsvFields = tsvFields;
}
public Resource getProjectsFundingQueryTemplate() {
return projectsFundingQueryTemplate;
}
public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) {
this.projectsFundingQueryTemplate = projectsFundingQueryTemplate;
}
public Resource getDspaceTemplate() {
return dspaceTemplate;
}
public void setDspaceTemplate(final Resource dspaceTemplate) {
this.dspaceTemplate = dspaceTemplate;
}
public Resource getDspaceHeadTemplate() {
return dspaceHeadTemplate;
}
public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) {
this.dspaceHeadTemplate = dspaceHeadTemplate;
}
public Resource getDspaceTailTemplate() {
return dspaceTailTemplate;
}
public void setDspaceTailTemplate(final Resource dspaceTailTemplate) {
this.dspaceTailTemplate = dspaceTailTemplate;
}
public Resource getEprintsTemplate() {
return eprintsTemplate;
}
public void setEprintsTemplate(final Resource eprintsTemplate) {
this.eprintsTemplate = eprintsTemplate;
}
}
public static class Jdbc {
// JDBC
@Value("${spring.datasource.driverClassName}")
private String driverClassName;
private String url;
private String user;
private String pwd;
private int minIdle;
private int maxidle;
private int maxRows;
public String getDriverClassName() {
return driverClassName;
}
public String getUrl() {
return url;
}
public void setUrl(final String url) {
this.url = url;
}
public String getUser() {
return user;
}
public void setUser(final String user) {
this.user = user;
}
public String getPwd() {
return pwd;
}
public void setPwd(final String pwd) {
this.pwd = pwd;
}
public int getMinIdle() {
return minIdle;
}
public void setMinIdle(final int minIdle) {
this.minIdle = minIdle;
}
public int getMaxidle() {
return maxidle;
}
public void setMaxidle(final int maxidle) {
this.maxidle = maxidle;
}
public int getMaxRows() {
return maxRows;
}
public void setMaxRows(final int maxRows) {
this.maxRows = maxRows;
}
}
public static class Swagger {
private String apiTitle;
private String apiDescription;
private String apiLicense;
private String apiLicenseUrl;
private String apiContactName;
private String apiContactUrl;
private String apiContactEmail;
public String getApiTitle() {
return apiTitle;
}
public void setApiTitle(final String apiTitle) {
this.apiTitle = apiTitle;
}
public String getApiDescription() {
return apiDescription;
}
public void setApiDescription(final String apiDescription) {
this.apiDescription = apiDescription;
}
public String getApiLicense() {
return apiLicense;
}
public void setApiLicense(final String apiLicense) {
this.apiLicense = apiLicense;
}
public String getApiLicenseUrl() {
return apiLicenseUrl;
}
public void setApiLicenseUrl(final String apiLicenseUrl) {
this.apiLicenseUrl = apiLicenseUrl;
}
public String getApiContactName() {
return apiContactName;
}
public void setApiContactName(final String apiContactName) {
this.apiContactName = apiContactName;
}
public String getApiContactUrl() {
return apiContactUrl;
}
public void setApiContactUrl(final String apiContactUrl) {
this.apiContactUrl = apiContactUrl;
}
public String getApiContactEmail() {
return apiContactEmail;
}
public void setApiContactEmail(final String apiContactEmail) {
this.apiContactEmail = apiContactEmail;
}
}
public static class Vocabularies {
private String baseUrl;
private String countriesEndpoint;
private String datasourceTypologiesEndpoint;
public String getCountriesEndpoint() {
return countriesEndpoint;
}
public void setCountriesEndpoint(final String countriesEndpoint) {
this.countriesEndpoint = countriesEndpoint;
}
public String getBaseUrl() {
return baseUrl;
}
public void setBaseUrl(final String baseUrl) {
this.baseUrl = baseUrl;
}
public String getDatasourceTypologiesEndpoint() {
return datasourceTypologiesEndpoint;
}
public void setDatasourceTypologiesEndpoint(final String datasourceTypologiesEndpoint) {
this.datasourceTypologiesEndpoint = datasourceTypologiesEndpoint;
}
}
public ClassPathResource getFindSolrIndexUrl() {
return findSolrIndexUrl;
}
public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) {
this.findSolrIndexUrl = findSolrIndexUrl;
}
public ClassPathResource getFindIndexDsInfo() {
return findIndexDsInfo;
}
public ClassPathResource getFindObjectStore() {
return findObjectStore;
}
public void setFindObjectStore(final ClassPathResource findObjectStore) {
this.findObjectStore = findObjectStore;
}
public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) {
this.findIndexDsInfo = findIndexDsInfo;
}
public ClassPathResource getFindFunderContexts() {
return findFunderContexts;
}
public void setFindFunderContexts(final ClassPathResource findFunderContexts) {
this.findFunderContexts = findFunderContexts;
}
public ClassPathResource getFindCommunityContexts() {
return findCommunityContexts;
}
public ClassPathResource getFindContextProfiles() {
return findContextProfiles;
}
public ClassPathResource getFindContextProfilesByType() {
return findContextProfilesByType;
}
public void setFindContextProfiles(final ClassPathResource findContextProfiles) {
this.findContextProfiles = findContextProfiles;
}
public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) {
this.findContextProfilesByType = findContextProfilesByType;
}
public void setFindCommunityContexts(final ClassPathResource findCommunityContexts) {
this.findCommunityContexts = findCommunityContexts;
}
public ClassPathResource getGetRepoProfile() {
return getRepoProfile;
}
public void setGetRepoProfile(final ClassPathResource getRepoProfile) {
this.getRepoProfile = getRepoProfile;
}
public String getContentLoadQuery() {
return contentLoadQuery;
}
public void setContentLoadQuery(String contentLoadQuery) {
this.contentLoadQuery = contentLoadQuery;
}
public String getIsLookupUrl() {
return isLookupUrl;
}
public void setIsLookupUrl(final String isLookupUrl) {
this.isLookupUrl = isLookupUrl;
}
public String getObjectStoreServiceUrl() {
return objectStoreServiceUrl;
}
public void setObjectStoreServiceUrl(final String objectStoreServiceUrl) {
this.objectStoreServiceUrl = objectStoreServiceUrl;
}
public String getIsRegistryServiceUrl() {
return isRegistryServiceUrl;
}
public void setIsRegistryServiceUrl(final String isRegistryServiceUrl) {
this.isRegistryServiceUrl = isRegistryServiceUrl;
}
public int getRequestWorkers() {
return requestWorkers;
}
public void setRequestWorkers(final int requestWorkers) {
this.requestWorkers = requestWorkers;
}
public int getRequestTimeout() {
return requestTimeout;
}
public void setRequestTimeout(final int requestTimeout) {
this.requestTimeout = requestTimeout;
}
public int getCxfClientConnectTimeout() {
return cxfClientConnectTimeout;
}
public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) {
this.cxfClientConnectTimeout = cxfClientConnectTimeout;
}
public int getCxfClientReceiveTimeout() {
return cxfClientReceiveTimeout;
}
public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) {
this.cxfClientReceiveTimeout = cxfClientReceiveTimeout;
}
public Datasource getDatasource() {
return datasource;
}
public void setDatasource(final Datasource datasource) {
this.datasource = datasource;
}
public Project getProject() {
return project;
}
public void setProject(final Project project) {
this.project = project;
}
public Jdbc getJdbc() {
return jdbc;
}
public void setJdbc(final Jdbc jdbc) {
this.jdbc = jdbc;
}
public Swagger getSwaggerDsm() {
return swaggerDsm;
}
public void setSwaggerDsm(final Swagger swaggerDsm) {
this.swaggerDsm = swaggerDsm;
}
public Swagger getSwaggerProjects() {
return swaggerProjects;
}
public void setSwaggerProjects(final Swagger swaggerProjects) {
this.swaggerProjects = swaggerProjects;
}
public Swagger getSwaggerFunders() {
return swaggerFunders;
}
public void setSwaggerFunders(final Swagger swaggerFunders) {
this.swaggerFunders = swaggerFunders;
}
public Swagger getSwaggerCommunities() {
return swaggerCommunities;
}
public void setSwaggerCommunities(final Swagger swaggerCommunities) {
this.swaggerCommunities = swaggerCommunities;
}
public Swagger getSwaggerContexts() {
return swaggerContexts;
}
public void setSwaggerContexts(final Swagger swaggerContexts) {
this.swaggerContexts = swaggerContexts;
}
public Swagger getSwaggerInfo() { return swaggerInfo; }
public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; }
public Vocabularies getVocabularies() {
return vocabularies;
}
public void setVocabularies(final Vocabularies vocabularies) {
this.vocabularies = vocabularies;
}
}

16
apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java Normal file
View File

@ -0,0 +1,16 @@
package eu.dnetlib;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
@Controller
public class SwaggerController {
@RequestMapping(value = {
"/", "/docs", "swagger-ui.html"
})
public String index() {
return "redirect:swagger-ui/";
}
}

100
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java Normal file
View File

@ -0,0 +1,100 @@
package eu.dnetlib.openaire.common;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
/**
* Created by claudio on 18/07/2017.
*/
public abstract class AbstractExporterController {
private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM
@ResponseBody
@ExceptionHandler({DsmException.class})
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public ErrorMessage handleDSMException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(DsmForbiddenException.class)
@ResponseStatus(value = HttpStatus.FORBIDDEN)
public ErrorMessage handleForbiddenException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler({DsmNotFoundException.class})
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public ErrorMessage handleNotFoundException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseStatus(HttpStatus.BAD_REQUEST)
public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
return e.getBindingResult()
.getFieldErrors().stream()
.map(fe -> new ErrorMessage(
String.format("field '%s'", fe.getField()),
String.format("rejected value '%s'", fe.getRejectedValue()),
fe.getDefaultMessage()))
.collect(Collectors.toList());
}
private ErrorMessage _handleError(final Exception e) {
log.error(e);
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
return null; //socket is closed, cannot return any response
} else {
return new ErrorMessage(e);
}
}
@JsonAutoDetect
public class ErrorMessage {
private final String message;
private final String details;
private final String stacktrace;
public ErrorMessage(final Exception e) {
this(e.getMessage(), "", ExceptionUtils.getStackTrace(e));
}
public ErrorMessage(final String message, final String details, final String stacktrace) {
this.message = message;
this.details = details;
this.stacktrace = stacktrace;
}
public String getMessage() {
return this.message;
}
public String getStacktrace() {
return this.stacktrace;
}
public String getDetails() {
return details;
}
}
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.common;
import java.sql.Array;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import javax.sql.DataSource;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
/**
* Created by claudio on 05/07/2017.
*/
@Converter
public class ConverterTextArray implements AttributeConverter<List<String>, Array>, ApplicationContextAware {
private ApplicationContext applicationContext;
@Override
public Array convertToDatabaseColumn(List<String> attribute) {
final Map<String, DataSource> datasources = applicationContext.getBeansOfType(DataSource.class);
DataSource source = datasources.values().stream().findFirst().get();
try {
Connection conn = source.getConnection();
return conn.createArrayOf("text", attribute.toArray());
} catch (SQLException e) {
e.printStackTrace();
}
return null;
}
@Override
public List<String> convertToEntityAttribute(Array dbData) {
try {
return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList());
} catch (SQLException e) {
e.printStackTrace();
return null;
}
}
@Override
public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
}

30
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java Normal file
View File

@ -0,0 +1,30 @@
package eu.dnetlib.openaire.common;
public class ExporterConstants {
/*
* Tags used to group the operations on the swagger UI
*/
public final static String C = "Community";
public final static String C_CP = "Community content providers";
public final static String C_PJ = "Community projects";
public final static String C_ZC = "Community Zenodo Communities";
public final static String C_O = "Community Organizations";
public final static String DS = "Datasource";
public final static String API = "Interface";
public final static String R = "Read";
public final static String W = "Write";
public final static String D = "Deprecated";
public final static String M = "Management";
public final static String DSPACE = "DSpace";
public final static String EPRINT = "EPrints";
public final static String TSV = "TSV";
public final static String STREAMING = "Streaming";
public static final String OAI = "oai";
public static final String SET = "set";
}

106
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java Normal file
View File

@ -0,0 +1,106 @@
package eu.dnetlib.openaire.common;
import java.io.Serializable;
import java.sql.*;
import org.hibernate.HibernateException;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.usertype.UserType;
/**
* Created by claudio on 05/07/2017.
*/
public class GenericArrayUserType<T extends Serializable> implements UserType {
protected static final int[] SQL_TYPES = { Types.ARRAY };
private Class<T> typeParameterClass;
@Override
public Object assemble(Serializable cached, Object owner) throws HibernateException {
return this.deepCopy(cached);
}
@Override
public Object deepCopy(Object value) throws HibernateException {
return value;
}
@SuppressWarnings("unchecked")
@Override
public Serializable disassemble(Object value) throws HibernateException {
return (T) this.deepCopy(value);
}
@Override
public boolean equals(Object x, Object y) throws HibernateException {
if (x == null) {
return y == null;
}
return x.equals(y);
}
@Override
public int hashCode(Object x) throws HibernateException {
return x.hashCode();
}
@Override
public Object nullSafeGet(final ResultSet resultSet,
final String[] names,
final SharedSessionContractImplementor sharedSessionContractImplementor,
final Object o)
throws HibernateException, SQLException {
if (resultSet.wasNull()) {
return null;
}
if (resultSet.getArray(names[0]) == null) {
return new Integer[0];
}
Array array = resultSet.getArray(names[0]);
@SuppressWarnings("unchecked")
T javaArray = (T) array.getArray();
return javaArray;
}
@Override
public void nullSafeSet(final PreparedStatement statement,
final Object value,
final int index,
final SharedSessionContractImplementor session)
throws HibernateException, SQLException {
Connection connection = statement.getConnection();
if (value == null) {
statement.setNull(index, SQL_TYPES[0]);
} else {
@SuppressWarnings("unchecked")
T castObject = (T) value;
Array array = connection.createArrayOf("integer", (Object[]) castObject);
statement.setArray(index, array);
}
}
@Override
public boolean isMutable() {
return true;
}
@Override
public Object replace(Object original, Object target, Object owner) throws HibernateException {
return original;
}
@Override
public Class<T> returnedClass() {
return typeParameterClass;
}
@Override
public int[] sqlTypes() {
return new int[] { Types.ARRAY };
}
}

48
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.openaire.common;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
public interface ISClient {
IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException;
String getObjectStoreId(String dsId) throws DsmException;
Map<String, Context> getFunderContextMap() throws IOException;
Map<String, Context> getCommunityContextMap() throws IOException;
Map<String, Context> getContextMap(final List<String> type) throws IOException;
void updateContextParam(String id, String name, String value);
void updateContextAttribute(String id, String name, String value);
void addConcept(String id, String categoryId, String data);
void removeConcept(String id, String categoryId, String conceptId);
void dropCache();
/**
*
* @param id
* id of the concept to be updated (i.e. ni::projects::2)
* @param name
* name of the attribute to be updated
* @param value
* new value for the attribute
*/
void updateConceptAttribute(String id, String name, String value);
void updateConceptParam(String id, String name, String value);
void updateConceptParamNoEscape(String id, String name, String value);
}

297
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java Normal file
View File

@ -0,0 +1,297 @@
package eu.dnetlib.openaire.common;
import static eu.dnetlib.openaire.common.Utils.escape;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.io.ClassPathResource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.google.common.collect.Lists;
import com.google.common.escape.Escaper;
import com.google.common.xml.XmlEscapers;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmRuntimeException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.ContextMappingUtils;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
/**
* Created by claudio on 20/10/2016.
*/
@Component
public class ISClientImpl implements ISClient {
private static final Log log = LogFactory.getLog(ISClientImpl.class);
@Autowired
private DnetOpenaireExporterProperties config;
@Autowired
private ISLookUpService isLookUpService;
@Override
@Cacheable("indexdsinfo-cache")
public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException {
log.warn("calculateCurrentIndexDsInfo(): not using cache");
final String[] arr;
try {
arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@");
return new IndexDsInfo(
_isLookUp(_getQuery(config.getFindSolrIndexUrl())),
arr[0].trim(), arr[1].trim(), arr[2].trim());
} catch (IOException | ISLookUpException e) {
throw new DsmException("unable fetch index DS information from IS");
}
}
@Override
@Cacheable("objectstoreid-cache")
public String getObjectStoreId(final String dsId) throws DsmException {
log.warn(String.format("getObjectStoreId(%s): not using cache", dsId));
try {
final String xqueryTemplate = _getQuery(config.getFindObjectStore());
return _isLookUp(String.format(xqueryTemplate, dsId));
} catch (IOException | ISLookUpException e) {
throw new DsmException("unble to find objectstore for ds " + dsId);
}
}
@Override
@Cacheable("context-cache-funder")
public Map<String, Context> getFunderContextMap() throws IOException {
return _processContext(_getQuery(config.getFindFunderContexts()));
}
@Override
@Cacheable("context-cache-community")
public Map<String, Context> getCommunityContextMap() throws IOException {
return _processContext(_getQuery(config.getFindCommunityContexts()));
}
@Override
@Cacheable("context-cache")
public Map<String, Context> getContextMap(final List<String> type) throws IOException {
if (Objects.isNull(type) || type.isEmpty()) {
return _processContext(_getQuery(config.getFindContextProfiles()));
} else {
final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
final String xquery = String.format(xqueryTemplate, type.stream()
.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
.collect(Collectors.joining(" or ")));
return _processContext(xquery);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateContextParam(final String id, final String name, final String value) {
try {
_quickSeachProfile(getXQuery(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateContextAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
_quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void addConcept(final String id, final String categoryId, final String data) {
try {
_quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void removeConcept(final String id, final String categoryId, final String conceptId) {
try {
_quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" +
"/category[./@id = '%s']/concept[./@id = '%s'] " +
"return update delete $concept", id, categoryId, conceptId));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-community", "context-cache-funder"
}, allEntries = true)
public void updateConceptAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
_quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value)));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateConceptParam(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQuery(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
@Override
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
public void updateConceptParamNoEscape(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
} catch (final ISLookUpException e) {
throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e);
}
}
/// HELPERS
private String getXQuery(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlContentEscaper();
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, escape(esc, value));
} else {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
"/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private String getConceptXQuery(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlContentEscaper();
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
"concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, escape(esc, value));
} else {
return String
.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
if (StringUtils.isNotBlank(value)) {
return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" +
"concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'>%s</param>", id, name, name, value);
} else {
return String
.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with <param name='%s'/>", id, name, name);
}
}
private Map<String, Context> _processContext(final String xquery) throws IOException {
return _processContext(new LinkedBlockingQueue<>(), xquery);
}
private Map<String, Context> _processContext(final Queue<Throwable> errors, final String xquery) throws IOException {
try {
return getContextProfiles(errors, xquery).stream()
.filter(StringUtils::isNotBlank)
.map(s -> ContextMappingUtils.parseContext(s, errors))
.collect(Collectors.toMap(Context::getId, Function.identity(), (c1, c2) -> {
log.warn(String.format("found duplicate context profile '%s'", c1.getId()));
return c1;
}));
} finally {
if (!errors.isEmpty()) {
log.error(errors);
errors.forEach(Throwable::printStackTrace);
}
}
}
private List<String> getContextProfiles(final Queue<Throwable> errors, final String xquery) throws IOException {
log.warn("getContextProfiles(): not using cache");
try {
return _quickSeachProfile(xquery);
} catch (final ISLookUpException e) {
throw new DsmRuntimeException("unable to get context profiles", e);
}
}
private String _getQuery(final ClassPathResource resource) throws IOException {
return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset());
}
private String _isLookUp(final String xquery) throws ISLookUpException {
log.debug(String.format("running xquery:\n%s", xquery));
// log.debug(String.format("query result: %s", res));
return isLookUpService.getResourceProfileByQuery(xquery);
}
private List<String> _quickSeachProfile(final String xquery) throws ISLookUpException {
final List<String> res = Lists.newArrayList();
log.debug(String.format("running xquery:\n%s", xquery));
final List<String> list = isLookUpService.quickSearchProfile(xquery);
if (list != null) {
res.addAll(list);
}
log.debug(String.format("query result size: %s", res.size()));
return res;
}
@Override
@CacheEvict(cacheNames = {
"context-cache", "indexdsinfo-cache", "objectstoreid-cache"
}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped dsManager IS cache");
}
}

60
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java Normal file
View File

@ -0,0 +1,60 @@
package eu.dnetlib.openaire.common;
import java.util.List;
import java.util.concurrent.*;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;
@Component
public class OperationManager {
private static final Log log = LogFactory.getLog(OperationManager.class);
private static final long SLEEP_TIME = 1000;
private static final int Q_SIZE = 100;
private static final int POOL_SIZE = 5;
private final BlockingQueue<Runnable> ops = new ArrayBlockingQueue<>(Q_SIZE);
private ExecutorService executor;
@PostConstruct
public void init() {
executor = getExecutor();
}
public int dropAll() {
final List<Runnable> lostOperations = executor.shutdownNow();
log.warn(String.format("discarding %s operations", lostOperations.size()));
executor = getExecutor();
return lostOperations.size();
}
public int getOpSize() {
return ops.size();
}
public void addOperation(final Runnable op) {
executor.execute(op);
}
@PreDestroy
public void tearDown() throws InterruptedException {
executor.shutdown();
final boolean done = executor.awaitTermination(SLEEP_TIME, TimeUnit.MILLISECONDS);
log.debug(String.format("All operations were completed so far? %s", done));
}
// HELPERS
private ThreadPoolExecutor getExecutor() {
return new ThreadPoolExecutor(POOL_SIZE, POOL_SIZE,0L, TimeUnit.MILLISECONDS, ops);
}
}

63
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java Normal file
View File

@ -0,0 +1,63 @@
package eu.dnetlib.openaire.common;
import java.text.FieldPosition;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.TimeZone;
import com.fasterxml.jackson.databind.util.StdDateFormat;
public class RFC3339DateFormat extends StdDateFormat {
/**
*
*/
private static final long serialVersionUID = 8174507696046505992L;
private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC");
// Same as ISO8601DateFormat but serializing milliseconds.
@Override
public StringBuffer format(final Date date, final StringBuffer toAppendTo, final FieldPosition fieldPosition) {
final String value = format(date, true, TIMEZONE_Z, Locale.US);
toAppendTo.append(value);
return toAppendTo;
}
/**
* Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
*
* @param date
* the date to format
* @param millis
* true to include millis precision otherwise false
* @param tz
* timezone to use for the formatting (UTC will produce 'Z')
* @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm]
*/
private static String format(final Date date, final boolean millis, final TimeZone tz, final Locale loc) {
final Calendar calendar = new GregorianCalendar(tz, loc);
calendar.setTime(date);
// estimate capacity of buffer as close as we can (yeah, that's pedantic ;)
final StringBuilder sb = new StringBuilder(30);
sb.append(String.format("%04d-%02d-%02dT%02d:%02d:%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar
.get(Calendar.DAY_OF_MONTH), calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND)));
if (millis) {
sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND)));
}
final int offset = tz.getOffset(calendar.getTimeInMillis());
if (offset != 0) {
final int hours = Math.abs(offset / (60 * 1000) / 60);
final int minutes = Math.abs(offset / (60 * 1000) % 60);
sb.append(String.format("%c%02d:%02d", offset < 0 ? '-' : '+', hours, minutes));
} else {
sb.append('Z');
}
return sb.toString();
}
}

22
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java Normal file
View File

@ -0,0 +1,22 @@
package eu.dnetlib.openaire.common;
import java.util.Iterator;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import com.google.common.escape.Escaper;
import org.apache.commons.lang3.StringUtils;
public class Utils {
public static <T> Stream<T> stream(Iterator<T> iterator) {
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
}
public static String escape(final Escaper esc, final String value) {
return StringUtils.isNotBlank(value) ? esc.escape(value) : "";
}
}

304
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java Normal file
View File

@ -0,0 +1,304 @@
package eu.dnetlib.openaire.community;
import java.util.List;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.*;
import static eu.dnetlib.openaire.common.ExporterConstants.*;
@RestController
@CrossOrigin(origins = { "*" })
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
@io.swagger.annotations.Api(tags = "OpenAIRE Communities API", description = "the OpenAIRE Community API")
public class CommunityApiController {
@Autowired
private CommunityApiCore communityApiCore;
@RequestMapping(value = "/community/communities", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get all community profiles",
notes = "get all community profiles",
tags = { C, R },
response = CommunitySummary[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunitySummary[].class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public List<CommunitySummary> listCommunities() throws CommunityException {
return communityApiCore.listCommunities();
}
@RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get community profile",
notes = "get community profile",
tags = { C, R },
response = CommunityDetails.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunityDetails.class),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityDetails getCommunity(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunity(id);
}
@RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "update community details",
notes = "update community details",
tags = { C, R })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public void setCommunity(
@PathVariable final String id,
@RequestBody CommunityWritableProperties properties) throws CommunityException, CommunityNotFoundException {
communityApiCore.setCommunity(id, properties);
}
@RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get community projects",
notes = "get community projects",
tags = { C_PJ, R },
response = CommunityProject[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunityProject[].class),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public List<CommunityProject> getCommunityProjects(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityProjects(id);
}
@RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "associate a project to the community",
notes = "associate a project to the community",
tags = { C_PJ, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityProject addCommunityProject(
@PathVariable final String id,
@RequestBody final CommunityProject project) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityProject(id, project);
}
@RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.DELETE)
@ApiOperation(
value = "remove a project from the community",
notes = "remove a project from the community",
tags = { C_PJ, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public void deleteCommunityProject(
@PathVariable final String id,
@RequestBody final Integer projectId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityProject(id, projectId);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get the list of content providers associated to a given community",
notes = "get the list of content providers associated to a given community",
tags = { C_CP, R },
response = CommunityContentprovider[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public List<CommunityContentprovider> getCommunityContentproviders(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityContentproviders(id);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "associate a content provider to the community",
notes = "associate a content provider to the community",
tags = { C_CP, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityContentprovider addCommunityContentprovider(
@PathVariable final String id,
@RequestBody final CommunityContentprovider contentprovider) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityContentprovider(id, contentprovider);
}
@RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.DELETE)
@ApiOperation(
value = "remove the association between a content provider and the community",
notes = "remove the association between a content provider and the community",
tags = { C_CP, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public void removeCommunityContentprovider(
@PathVariable final String id,
@RequestBody final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityContentProvider(id, contentproviderId);
}
//ADDING CODE FOR COMMUNITY ORGANIZATIONS
@RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get the list of organizations for a given community",
notes = "get the list of organizations for a given community",
tags = { C_O, R },
response = CommunityOrganization[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public List<CommunityOrganization> getCommunityOrganizations(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityOrganizations(id);
}
@RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "associate an organization to the community",
notes = "associate an organization to the community",
tags = { C_O, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityOrganization addCommunityOrganization(
@PathVariable final String id,
@RequestBody final CommunityOrganization organization) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityOrganization(id, organization);
}
@RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.DELETE)
@ApiOperation(
value = "remove the association between an organization and the community",
notes = "remove the association between an organization and the community",
tags = { C_O, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public void removeCommunityOrganization(
@PathVariable final String id,
@RequestBody final Integer organizationId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityOrganization(id, organizationId);
}
//**********************
@RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "associate a subject to the community",
notes = "associate a subject to the community",
tags = { C, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityDetails addCommunitySubjects(
@PathVariable final String id,
@RequestBody final List<String> subjects) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunitySubjects(id, subjects);
}
@RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.DELETE)
@ApiOperation(
value = "remove subjects from a community",
notes = "remove subjects from a community",
tags = { C, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityDetails removeCommunitySubjects(
@PathVariable final String id,
@RequestBody final List<String> subjects) throws CommunityException, CommunityNotFoundException {
return communityApiCore.removeCommunitySubjects(id, subjects);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get the list of Zenodo communities associated to a given community",
notes = "get the list of Zenodo communities associated to a given community",
tags = { C_ZC, R },
response = CommunityZenodoCommunity[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CommunityZenodoCommunity[].class),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(@PathVariable final String id) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getCommunityZenodoCommunities(id);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.POST)
@ApiOperation(
value = "associate a Zenodo community to the community",
notes = "associate a Zenodo community to the community",
tags = { C_ZC, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityZenodoCommunity addCommunityZenodoCommunity(
@PathVariable final String id,
@RequestBody final CommunityZenodoCommunity zenodocommunity) throws CommunityException, CommunityNotFoundException {
return communityApiCore.addCommunityZenodoCommunity(id, zenodocommunity);
}
@RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.DELETE)
@ApiOperation(
value = "remove a Zenodo community from a community",
notes = "remove a Zenodo community from a community",
tags = { C_ZC, W })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public void removeCommunityZenodoCommunity(
@PathVariable final String id,
@RequestBody final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
communityApiCore.removeCommunityZenodoCommunity(id, zenodoCommId);
}
@RequestMapping(value = "/community/{zenodoId}/openairecommunities", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "get the list of OpenAIRE communities associated to a given Zenodo community",
notes = "get the list of OpenAIRE communities associated to a given Zenodo community",
tags = { C_ZC, R })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class),
@ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) })
public CommunityOpenAIRECommunities getOpenAireCommunities(
@PathVariable final String zenodoId) throws CommunityException, CommunityNotFoundException {
return communityApiCore.getOpenAIRECommunities(zenodoId);
}
}

368
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java Normal file
View File

@ -0,0 +1,368 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_OFFICIALNAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_SELCRITERIA;
import static eu.dnetlib.openaire.community.CommunityConstants.CLABEL;
import static eu.dnetlib.openaire.community.CommunityConstants.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_WEBSITEURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SUBJECT;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_ACRONYM;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FULLNAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FUNDER;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_NUMBER;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_DESCRIPTION;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_STATUS;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_ZENODOC;
import static eu.dnetlib.openaire.community.CommunityConstants.CSV_DELIMITER;
import static eu.dnetlib.openaire.community.CommunityConstants.ID_SEPARATOR;
import static eu.dnetlib.openaire.community.CommunityConstants.OPENAIRE_ID;
import static eu.dnetlib.openaire.community.CommunityConstants.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.PROJECTS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.ZENODOCOMMUNITY_ID_SUFFIX;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Component;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.openaire.common.ISClient;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
public class CommunityApiCore {// implements CommunityClient{
private static final Log log = LogFactory.getLog(CommunityApiCore.class);
@Autowired
private CommunityClient cci;
@Autowired
private ISClient isClient;
@Autowired
private CommunityCommon cc;
public List<CommunitySummary> listCommunities() throws CommunityException {
return cc.listCommunities();
}
public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
return cc.getCommunity(id);
}
public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
if (details.getShortName() != null) {
isClient.updateContextAttribute(id, CLABEL, details.getShortName());
}
if (details.getName() != null) {
isClient.updateContextParam(id, CSUMMARY_NAME, details.getName());
}
if (details.getDescription() != null) {
isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription());
}
if (details.getLogoUrl() != null) {
isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl());
}
if (details.getStatus() != null) {
isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name());
}
if (details.getSubjects() != null) {
isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()));
}
if (details.getMainZenodoCommunity() != null) {
isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity());
}
cc.updateCommunity(id, details);
}
public List<CommunityProject> getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
}
public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) {
throw new CommunityException("parameters 'id' and project.communityId must be coherent");
}
final TreeMap<Integer, CommunityProject> projects = getCommunityProjectMap(id);
final String project_id = project.getId();
if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))) {
if (project.getName() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName());
}
if (project.getAcronym() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym());
}
if (project.getOpenaireId() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId());
}
if (project.getFunder() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder());
}
if (project.getGrantId() != null) {
isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId());
}
} else {
project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0));
isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project));
}
cc.updateProject(id, project);
return project;
}
private String nextId(final Integer id) {
return String.valueOf(id + 1);
}
public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityProject> projects = getCommunityProjectMap(id);
if (!projects.containsKey(projectId)) {
throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id));
}
isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId);
cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId));
}
public List<CommunityContentprovider> getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
}
public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp)
throws CommunityException, CommunityNotFoundException {
log.info("content provider to add " + cp.toString());
if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); }
final TreeMap<Integer, CommunityContentprovider> cps = getCommunityContentproviderMap(id);
final String concept_id = cp.getId();
if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) {
if (cp.getName() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName());
}
if (cp.getOfficialname() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname());
}
if (cp.getOpenaireId() != null) {
isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId());
}
if (cp.getSelectioncriteria() != null) {
isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML());
}
} else {
log.info("adding new concept for community " + id);
cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp));
}
cc.updateDatasource(id, cp);
return cp;
}
public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityContentprovider> providers = getCommunityContentproviderMap(id);
if (!providers.containsKey(contentproviderId)) {
throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id));
}
isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId);
cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId));
}
public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityOrganization> organizations = getCommunityOrganizationMap(id);
if (!organizations.containsKey(organizationId)) {
throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id));
}
isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId);
cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId));
}
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
return cc.getCommunityZenodoCommunities(id);
}
public List<CommunityOrganization> getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException {
cc.getCommunity(id);
return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
}
public CommunityDetails addCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, CommunityNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.addAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, CommunityNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.removeAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
@CacheEvict(value = "community-cache", allEntries = true)
public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException {
final Map<Integer, CommunityZenodoCommunity> zcomms = getZenodoCommunityMap(id);
if (!zcomms.containsKey(zenodoCommId)) {
throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
}
isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
}
@CacheEvict(value = "community-cache", allEntries = true)
public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc)
throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); }
if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); }
final TreeMap<Integer, CommunityZenodoCommunity> zcs = getZenodoCommunityMap(id);
for (final CommunityZenodoCommunity czc : zcs.values()) {
if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); }
}
zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
cc.updateZenodoCommunity(id, zc);
return zc;
}
public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, CommunityNotFoundException {
if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) {
return new CommunityOpenAIRECommunities().setZenodoid(zenodoId)
.setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
}
return new CommunityOpenAIRECommunities();
}
// HELPERS
private TreeMap<Integer, CommunityProject> getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityProjects(id).stream()
.collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> {
log.warn(String.format("duplicate project found: '%s'", p1.getId()));
return p2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityContentprovider> getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException {
log.info("getting community content provider map");
return getCommunityContentproviders(id).stream()
.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
log.warn(String.format("duplicate content provider found: '%s'", cp1.getId()));
return cp2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityZenodoCommunity> getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityZenodoCommunities(id).stream()
.collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> {
log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId()));
return cp2;
}, TreeMap::new));
}
private TreeMap<Integer, CommunityOrganization> getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException {
return getCommunityOrganizations(id).stream()
.collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> {
log.warn(String.format("duplicate content provider found: '%s'", o1.getId()));
return o2;
}, TreeMap::new));
}
public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization)
throws CommunityException, CommunityNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) {
throw new CommunityException("parameters 'id' and organization.communityId must be coherent");
}
final TreeMap<Integer, CommunityOrganization> cps = getCommunityOrganizationMap(id);
final String organization_id = organization.getId();
if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) {
if (organization.getName() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName());
}
if (organization.getLogo_url() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder()
.encodeToString(organization.getLogo_url().getBytes()));
}
if (organization.getWebsite_url() != null) {
isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder()
.encodeToString(organization.getWebsite_url().getBytes()));
}
} else {
organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0));
isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization));
}
cc.updateOrganization(id, organization);
return organization;
}
}

12
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java Normal file
View File

@ -0,0 +1,12 @@
package eu.dnetlib.openaire.community;
import java.util.Map;
import java.util.Set;
public interface CommunityClient {
Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, CommunityNotFoundException;
void dropCache();
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.community;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.*;
@Component
public class CommunityClientImpl implements CommunityClient {
private static final Log log = LogFactory.getLog(CommunityClient.class);
@Autowired
private CommunityCommon communityCommon;
@Override
@Cacheable("community-cache")
public Map<String, Set<String>> getInverseZenodoCommunityMap () throws CommunityException, CommunityNotFoundException {
log.info("Creating the data structure. Not using cache");
final Map<String, Set<String>> inverseListMap = new HashMap<>();
final List<CommunitySummary> communityList = communityCommon.listCommunities();
for(CommunitySummary cs :communityList){
final String communityId = cs.getId();
List<CommunityZenodoCommunity> czc = communityCommon.getCommunityZenodoCommunities(communityId);
for(CommunityZenodoCommunity zc:czc){
final String zenodoId = zc.getZenodoid();
if(!inverseListMap.containsKey(zenodoId)) {
inverseListMap.put(zc.getZenodoid(),new HashSet<>());
}
inverseListMap.get(zc.getZenodoid()).add(communityId);
}
final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
if(!inverseListMap.containsKey(zenodoMainCommunity)) {
inverseListMap.put(zenodoMainCommunity,new HashSet<>());
}
inverseListMap.get(zenodoMainCommunity).add(communityId);
}
return inverseListMap;
}
@Override
@CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache(){
log.debug("dropped community cache");
}
}

438
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java Normal file
View File

@ -0,0 +1,438 @@
package eu.dnetlib.openaire.community;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.context.Category;
import eu.dnetlib.openaire.context.Concept;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.Param;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import static eu.dnetlib.openaire.community.CommunityConstants.*;
@Component
public class CommunityCommon {
@Autowired
private ISClient isClient;
public Map<String, Context> getContextMap() throws CommunityException {
try {
return isClient.getCommunityContextMap();
} catch (IOException e) {
throw new CommunityException(e);
}
}
public List<CommunitySummary> listCommunities() throws CommunityException {
return getContextMap().values().stream()
.filter(context -> !communityBlackList.contains(context.getId()))
.map(CommunityMappingUtils::asCommunitySummary)
.collect(Collectors.toList());
}
public <R> List<R> getCommunityInfo(final String id, final String idSuffix, final Function<Concept, R> mapping) throws CommunityException {
final Map<String, Context> contextMap = getContextMap();
final Context context = contextMap.get(id);
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(id + idSuffix);
if (category != null) {
return category.getConcepts().stream()
.map(mapping)
.collect(Collectors.toList());
}
}
return Lists.newArrayList();
}
public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException {
final Context context = getContextMap().get(id);
if (context == null || CommunityConstants.communityBlackList.contains(id)) {
throw new CommunityNotFoundException(String.format("community '%s' does not exist", id));
}
return CommunityMappingUtils.asCommunityProfile(context);
}
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException {
getCommunity(id); // ensure the community exists.
return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
}
public void updateProject(String communityId, CommunityProject project) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX);
if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){
prj.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) {
if (project.getName() != null) {
concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param()
.setName(CPROJECT_FULLNAME).setValue(project.getName())));
}
if (project.getAcronym() != null) {
if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){
concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param()
.setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
}
else{
concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param()
.setName(CPROJECT_ACRONYM).setValue(project.getAcronym())));
}
}
if (project.getOpenaireId() != null) {
if(concept.getParams().keySet().contains(OPENAIRE_ID)){
concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
}
else{
concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(project.getOpenaireId())));
}
}
if (project.getFunder() != null) {
concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param()
.setName(CPROJECT_FUNDER).setValue(project.getFunder())));
}
if (project.getGrantId() != null) {
concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param()
.setName(CPROJECT_NUMBER).setValue(project.getGrantId())));
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId());
concept.setClaim(false);
if(project.getAcronym() != null)
concept.setLabel(project.getAcronym());
else
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if(project.getAcronym() != null){
params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM)
.setValue(project.getAcronym())));
}
if (project.getName() != null){
params.put(CPROJECT_FULLNAME, Arrays.asList(new Param()
.setName(CPROJECT_FULLNAME)
.setValue(project.getName())
));
}
if (project.getOpenaireId() != null){
params.put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID)
.setValue(project.getOpenaireId())
));
}
if(project.getFunder() != null){
params.put(CPROJECT_FUNDER, Arrays.asList(new Param()
.setName(CPROJECT_FUNDER)
.setValue(project.getFunder())
));
}
if (project.getGrantId()!=null){
params.put(CPROJECT_NUMBER, Arrays.asList(new Param()
.setName(CPROJECT_NUMBER)
.setValue(project.getGrantId())
));
}
concept.setParams(params);
prj.getConcepts().add(concept);
}
}
public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException {
final Context context = getContextMap().get(id);
if(community.getShortName() != null) {
context.setLabel(community.getShortName());
}
if (community.getName() != null){
context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param()
.setValue(community.getName()).setName(CSUMMARY_NAME)));
}
if(community.getDescription() != null) {
context.getParams()
.replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
.setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription())));
}
if(community.getLogoUrl() != null){
context.getParams()
.replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
.setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl())));
}
if (community.getStatus() != null) {
context.getParams()
.replace(CSUMMARY_STATUS, Arrays.asList(new Param()
.setName(CSUMMARY_STATUS).setValue(community.getStatus().name())));
}
if (community.getSubjects() != null) {
context.getParams()
.replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getSubjects()))));
}
if(community.getMainZenodoCommunity() != null){
context.getParams()
.replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
.setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity())));
}
}
public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException {
Map<String, Context> cmap = getContextMap();
Context context = cmap.get(communityId);
Map<String, Category> cat = context.getCategories();
List<Concept> concepts = cat.get(communityId + category).getConcepts()
.stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList());
cat.get(communityId + category).setConcepts(concepts);
}
public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX);
if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){
dts.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) {
if (cp.getName() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){
concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_NAME).setValue(cp.getName())));
}
}
if (cp.getOfficialname() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){
concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname())));
}
}
if (cp.getOpenaireId() != null) {
if(concept.getParams().keySet().contains(OPENAIRE_ID)){
concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
}
else{
concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param()
.setName(OPENAIRE_ID).setValue(cp.getOpenaireId())));
}
}
if (cp.getSelectioncriteria() != null) {
if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){
concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
}
else{
concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param()
.setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson())));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId());
concept.setClaim(false);
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if (cp.getName() != null) {
params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME)));
}
if(cp.getOfficialname()!= null){
params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME)));
}
if (cp.getOpenaireId() != null){
params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID)));
}
if(cp.getSelectioncriteria() != null){
params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA)));
}
concept.setParams(params);
dts.getConcepts().add(concept);
}
}
public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX);
if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){
orgs.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) {
if (organization.getName() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){
concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param()
.setName(CORGANIZATION_NAME).setValue(organization.getName())));
}
else{
concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param()
.setName(CORGANIZATION_NAME).setValue(organization.getName())));
}
}
if (organization.getLogo_url() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){
concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
.setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
}
else{
concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param()
.setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))));
}
}
if (organization.getWebsite_url() != null) {
if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){
concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
.setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
}
else{
concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param()
.setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId());
concept.setClaim(false);
concept.setLabel("");
Map<String, List<Param>> params = new TreeMap<>();
if (organization.getName() != null) {
params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME)));
}
if(organization.getLogo_url()!= null){
params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL)));
}
if (organization.getWebsite_url() != null){
params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL)));
}
concept.setParams(params);
orgs.getConcepts().add(concept);
}
}
public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException {
final Context context = getContextMap().get(communityId);
Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList())
.contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){
zcs.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
if (zc.getZenodoid() != null) {
if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){
concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
}
else{
concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid())));
}
}
}
});
}
else{
Concept concept = new Concept();
concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
concept.setClaim(false);
Map<String, List<Param>> params = new TreeMap<>();
if (zc.getZenodoid() != null) {
params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
concept.setLabel(zc.getZenodoid());
}else{
concept.setLabel("");
}
concept.setParams(params);
zcs.getConcepts().add(concept);
}
}
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.community;
import java.util.Set;
import com.google.common.collect.Sets;
public class CommunityConstants {
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
// common
public final static String OPENAIRE_ID = "openaireId";
public final static String PIPE_SEPARATOR = "||";
public final static String ID_SEPARATOR = "::";
public final static String CSV_DELIMITER = ",";
public final static String CLABEL = "label";
// id suffixes
public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
public final static String CSUMMARY_DESCRIPTION = "description";
public final static String CSUMMARY_LOGOURL = "logourl";
public final static String CSUMMARY_STATUS = "status";
public final static String CSUMMARY_NAME = "name";
public final static String CSUMMARY_MANAGER = "manager";
public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
public final static String CPROFILE_SUBJECT = "subject";
public final static String CPROFILE_CREATIONDATE = "creationdate";
// community project
public final static String CPROJECT_FUNDER = "funder";
public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
public final static String CPROJECT_FULLNAME = "projectfullname";
public final static String CPROJECT_ACRONYM = "acronym";
// community content provider
public final static String CCONTENTPROVIDER_NAME = "name";
public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
public final static String CCONTENTPROVIDER_ENABLED = "enabled";
public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
//community zenodo community
public final static String CZENODOCOMMUNITY_ID = "zenodoid";
//community organization
public final static String CORGANIZATION_NAME = "name";
public final static String CORGANIZATION_LOGOURL = "logourl";
public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
}

100
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java Normal file
View File

@ -0,0 +1,100 @@
package eu.dnetlib.openaire.community;
import javax.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.gson.Gson;
import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunityContentprovider {
@ApiModelProperty(value = "OpenAIRE identifier for this content provider, if available", required = false)
private String openaireId;
@NotNull
@ApiModelProperty(value = "the community identifier this content provider belongs to", required = true)
private String communityId;
@NotNull
@ApiModelProperty(value = "identifies this content provider within the context it belongs to", required = true)
private String id;
@ApiModelProperty(value = "content provider name", required = false)
private String name;
@NotNull
@ApiModelProperty(value = "content provider official name", required = true)
private String officialname;
// @NotNull
@ApiModelProperty(value = "content provider selection criteria", required = false)
private SelectionCriteria selectioncriteria;
public String getOpenaireId() {
return openaireId;
}
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(final String communityId) {
this.communityId = communityId;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public SelectionCriteria getSelectioncriteria() {
return this.selectioncriteria;
}
public void setSelectioncriteria(final SelectionCriteria selectioncriteria) {
this.selectioncriteria = selectioncriteria;
}
@Override
public String toString() {
return String.format("id %s, name %s, selection criteria %s", this.id, this.name, toJson());
}
public String toJson() {
if (selectioncriteria == null) { return ""; }
return new Gson().toJson(selectioncriteria);
}
public String toXML() {
if (selectioncriteria == null) { return ""; }
return "<![CDATA[" + toJson() + "]]>";
}
}

51
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java Normal file
View File

@ -0,0 +1,51 @@
package eu.dnetlib.openaire.community;
import java.util.Date;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunityDetails extends CommunitySummary {
@ApiModelProperty("date of creation for this community")
private Date creationDate;
@ApiModelProperty("date of the last update for this communityu")
private Date lastUpdateDate;
@ApiModelProperty("list of subjects (keywords) that characterise this community")
private List<String> subjects;
public CommunityDetails() {
}
public CommunityDetails(final CommunitySummary summary) {
super(summary);
}
public Date getCreationDate() {
return creationDate;
}
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public List<String> getSubjects() {
return subjects;
}
public void setSubjects(final List<String> subjects) {
this.subjects = subjects;
}
public Date getLastUpdateDate() {
return lastUpdateDate;
}
public void setLastUpdateDate(Date lastUpdateDate) {
this.lastUpdateDate = lastUpdateDate;
}
}

25
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.community;
import java.io.IOException;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public class CommunityException extends Exception {
/**
*
*/
private static final long serialVersionUID = -4961233580574761346L;
public CommunityException(final String message) {
super(message);
}
public CommunityException(final IOException e) {
super(e);
}
}

231
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java Normal file
View File

@ -0,0 +1,231 @@
package eu.dnetlib.openaire.community;
import java.text.ParseException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.escape.Escaper;
import com.google.common.xml.XmlEscapers;
import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.context.Concept;
import eu.dnetlib.openaire.context.Context;
import eu.dnetlib.openaire.context.Param;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static eu.dnetlib.openaire.common.Utils.escape;
import static eu.dnetlib.openaire.community.CommunityConstants.*;
public class CommunityMappingUtils {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
public static CommunitySummary asCommunitySummary(final Context c) {
final CommunitySummary summary = new CommunitySummary();
summary.setId(c.getId());
summary.setShortName(c.getLabel());
summary.setLastUpdateDate(c.getLastUpdateDate());
summary.setCreationDate(c.getCreationDate());
summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
summary.setType(c.getType());
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
summary.setName(asCsv(params.get(CSUMMARY_NAME)));
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
return summary;
}
public static CommunityDetails asCommunityProfile(final Context c) {
final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
p.setLastUpdateDate(c.getLastUpdateDate());
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CPROFILE_SUBJECT)) {
p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_CREATIONDATE)){
try {
p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern));
}catch(ParseException e) {
e.printStackTrace();
}
}
return p;
}
public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){
final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
final Map<String, List<Param>> p = c.getParams();
z.setCommunityId(communityId);
z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID));
//z.setName(c.getLabel());
return z;
}
public static CommunityOrganization asCommunityOrganization(String id, Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p,CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL)));
return o;
}
private static String getDecodedUrl(final String encoded_url){
if(encoded_url == null){
return encoded_url;
}
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty() : params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
public static String asProjectXML(final String contextId, final CommunityProject project) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym())));
sb.append(paramXML(CPROJECT_FULLNAME, project.getName()));
sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym()));
sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId()));
sb.append(paramXML(CPROJECT_FUNDER, project.getFunder()));
sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId()));
sb.append("</concept>\n");
return sb.toString();
}
public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) {
log.info("creating the XML for the content provider");
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName())));
sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId()));
sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName()));
sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname()));
sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT));
sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML()));
sb.append("</concept>\n");
log.info(sb.toString());
return sb.toString();
}
public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
sb.append("</concept>\n");
return sb.toString();
}
public static String asOrganizationXML(final String contextId, CommunityOrganization organization) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(
String.format(
"<concept claim='false' id='%s%s%s%s' label='%s'>\n",
escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName())));
sb.append(paramXML(CORGANIZATION_NAME, organization.getName()));
sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())));
sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())));
sb.append("</concept>\n");
return sb.toString();
}
private static String paramXML(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
}
private static String paramXMLNoEscape(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, value);
}
}

24
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java Normal file
View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.community;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public class CommunityNotFoundException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5605421323034135778L;
public CommunityNotFoundException(final String msg) {
super(msg);
}
public CommunityNotFoundException(final Exception e) {
super(e);
}
}

42
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java Normal file
View File

@ -0,0 +1,42 @@
package eu.dnetlib.openaire.community;
import io.swagger.annotations.ApiModelProperty;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.List;
public class CommunityOpenAIRECommunities {
@NotNull
@ApiModelProperty(value = "the zenodo community identifier", required = true)
private String zenodoid;
@NotNull
@ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true)
private List<String> openAirecommunitylist;
public CommunityOpenAIRECommunities() {
this.zenodoid = "";
openAirecommunitylist=new ArrayList<>();
}
public List<String> getOpenAirecommunitylist() {
return openAirecommunitylist;
}
public CommunityOpenAIRECommunities setOpenAirecommunitylist(List<String> openAirecommunitylist) {
this.openAirecommunitylist = openAirecommunitylist;
return this;
}
public String getZenodoid() {
return zenodoid;
}
public CommunityOpenAIRECommunities setZenodoid(String zenodoid) {
this.zenodoid = zenodoid;
return this;
}
}

75
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java Normal file
View File

@ -0,0 +1,75 @@
package eu.dnetlib.openaire.community;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
import javax.validation.constraints.NotNull;
@JsonAutoDetect
public class CommunityOrganization {
@NotNull
@ApiModelProperty(value = "the community identifier this organization belongs to", required = true)
private String communityId;
@NotNull
@ApiModelProperty(value = "name of the organization", required = true)
private String name;
@NotNull
@ApiModelProperty(value = "identifies this organization within the context it belongs to", required = true)
private String id;
@NotNull
@ApiModelProperty(value="url of the logo for this organization", required = true)
private String logo_url;
@NotNull
@ApiModelProperty(value="website url for this organization", required = true)
private String website_url;
public String getCommunityId() {
return communityId;
}
public CommunityOrganization setCommunityId(String communityId) {
this.communityId = communityId;
return this;
}
public String getName() {
return name;
}
public CommunityOrganization setName(String name) {
this.name = name;
return this;
}
public String getId() {
return id;
}
public CommunityOrganization setId(String id) {
this.id = id;
return this;
}
public String getLogo_url() {
return logo_url;
}
public CommunityOrganization setLogo_url(String logo_url) {
this.logo_url = logo_url;
return this;
}
public String getWebsite_url() {
return website_url;
}
public CommunityOrganization setWebsite_url(String website_url) {
this.website_url = website_url;
return this;
}
}

86
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java Normal file
View File

@ -0,0 +1,86 @@
package eu.dnetlib.openaire.community;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunityProject {
@ApiModelProperty(value = "OpenAIRE identifier for this project, if available", required = false)
private String openaireId;
@ApiModelProperty(value = "the community identifier this project belongs to", required = true)
private String communityId;
@ApiModelProperty(value = "identifies this project within the context it belongs to", required = true)
private String id;
@ApiModelProperty(value = "project name", required = true)
private String name;
@ApiModelProperty(value = "project acronym", required = false)
private String acronym;
@ApiModelProperty(value = "project funder", required = true)
private String funder;
@ApiModelProperty(value = "project grant id", required = true)
private String grantId;
public String getOpenaireId() {
return openaireId;
}
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(final String communityId) {
this.communityId = communityId;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getAcronym() {
return acronym;
}
public void setAcronym(final String acronym) {
this.acronym = acronym;
}
public String getFunder() {
return funder;
}
public void setFunder(final String funder) {
this.funder = funder;
}
public String getGrantId() {
return grantId;
}
public void setGrantId(final String grantId) {
this.grantId = grantId;
}
}

17
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java Normal file
View File

@ -0,0 +1,17 @@
package eu.dnetlib.openaire.community;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public enum CommunityStatus {
@ApiModelProperty("restricted visibility")
hidden,
@ApiModelProperty("visible only to RCD managers")
manager,
@ApiModelProperty("visible to RCD managers and to the community users")
all
}

174
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java Normal file
View File

@ -0,0 +1,174 @@
package eu.dnetlib.openaire.community;
import java.util.Date;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunitySummary {
@ApiModelProperty("identifies the community")
protected String id;
@ApiModelProperty("values for this field reflect the index field _community_ in the index, e.g. 'egi||EGI Federation'")
protected String queryId;
@ApiModelProperty("community type")
protected String type;
@ApiModelProperty("community name")
protected String name;
@ApiModelProperty("community short name")
protected String shortName;
@ApiModelProperty("community creation date")
protected Date creationDate;
@ApiModelProperty("community last update date")
protected Date lastUpdateDate;
@ApiModelProperty("community description")
protected String description;
@ApiModelProperty("http url for the community logo")
protected String logoUrl;
@ApiModelProperty("status of the community, drives its visibility")
protected CommunityStatus status;
@ApiModelProperty("Zenodo community associated to this community")
protected String zenodoCommunity;
public CommunitySummary() {}
public CommunitySummary(
final String id,
final String queryId,
final String type,
final String name,
final String shortName,
final Date creationDate,
final Date lastUpdateDate,
final String description,
final String logoUrl,
final CommunityStatus status,
final String zenodoCommunity) {
this.id = id;
this.queryId = queryId;
this.type = type;
this.name = name;
this.shortName = shortName;
this.creationDate = creationDate;
this.lastUpdateDate = lastUpdateDate;
this.description = description;
this.logoUrl = logoUrl;
this.status = status;
this.zenodoCommunity = zenodoCommunity;
}
public CommunitySummary(final CommunitySummary summary) {
this(summary.getId(),
summary.getQueryId(),
summary.getType(),
summary.getName(),
summary.getShortName(),
summary.getCreationDate(),
summary.getLastUpdateDate(),
summary.getDescription(),
summary.getLogoUrl(),
summary.getStatus(),
summary.getZenodoCommunity());
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getQueryId() {
return queryId;
}
public void setQueryId(final String queryId) {
this.queryId = queryId;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getShortName() {
return shortName;
}
public void setShortName(final String shortName) {
this.shortName = shortName;
}
public Date getCreationDate() {
return creationDate;
}
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public Date getLastUpdateDate() {
return lastUpdateDate;
}
public void setLastUpdateDate(final Date lastUpdateDate) {
this.lastUpdateDate = lastUpdateDate;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public CommunityStatus getStatus() {
return status;
}
public void setStatus(final CommunityStatus status) {
this.status = status;
}
public String getZenodoCommunity() {
return zenodoCommunity;
}
public void setZenodoCommunity(final String zenodoCommunity) {
this.zenodoCommunity = zenodoCommunity;
}
}

96
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java Normal file
View File

@ -0,0 +1,96 @@
package eu.dnetlib.openaire.community;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunityWritableProperties {
@ApiModelProperty("community name")
private String name;
@ApiModelProperty("community short name")
private String shortName;
@ApiModelProperty("community description")
private String description;
@ApiModelProperty("http url for the community logo")
private String logoUrl;
@ApiModelProperty("list of subjects (keywords) that characterise this community")
private List<String> subjects;
@ApiModelProperty("status of the community, drives its visibility")
private CommunityStatus status;
@ApiModelProperty("id of the main Zenodo community")
private String mainZenodoCommunity;
public static CommunityWritableProperties fromDetails(final CommunityDetails details) {
CommunityWritableProperties p = new CommunityWritableProperties();
p.setName(details.getName());
p.setShortName(details.getShortName());
p.setDescription(details.getDescription());
p.setLogoUrl(details.getLogoUrl());
p.setSubjects(details.getSubjects());
p.setStatus(details.getStatus());
p.setMainZenodoCommunity(details.getZenodoCommunity());
return p;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getShortName() {
return shortName;
}
public void setShortName(final String shortName) {
this.shortName = shortName;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public List<String> getSubjects() {
return subjects;
}
public void setSubjects(final List<String> subjects) {
this.subjects = subjects;
}
public CommunityStatus getStatus() {
return status;
}
public void setStatus(final CommunityStatus status) {
this.status = status;
}
public String getMainZenodoCommunity() { return mainZenodoCommunity; }
public void setMainZenodoCommunity(String mainZenodoCommunity) { this.mainZenodoCommunity = mainZenodoCommunity; }
}

47
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java Normal file
View File

@ -0,0 +1,47 @@
package eu.dnetlib.openaire.community;
import javax.validation.constraints.NotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
public class CommunityZenodoCommunity {
@NotNull
@ApiModelProperty(value = "the community identifier this zenodo Community belongs to", required = true)
private String communityId;
@NotNull
@ApiModelProperty(value = "Zenodo identifier for this community", required = true)
private String zenodoid;
@NotNull
@ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true)
private String id;
public String getZenodoid() {
return zenodoid;
}
public void setZenodoid(String zenodoid) {
this.zenodoid = zenodoid;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getCommunityId() {
return communityId;
}
public void setCommunityId(String communityId) {
this.communityId = communityId;
}
}

45
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java Normal file
View File

@ -0,0 +1,45 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class Constraint implements Serializable {
/**
*
*/
private static final long serialVersionUID = -5996232267609464747L;
private String verb;
private String field;
private String value;
public Constraint() {}
public String getVerb() {
return verb;
}
public void setVerb(final String verb) {
this.verb = verb;
}
public String getField() {
return field;
}
public void setField(final String field) {
this.field = field;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
}

27
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java Normal file
View File

@ -0,0 +1,27 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class Constraints implements Serializable {
/**
*
*/
private static final long serialVersionUID = 2694950017620361195L;
private List<Constraint> constraint;
public Constraints() {}
public List<Constraint> getConstraint() {
return constraint;
}
public void setConstraint(final List<Constraint> constraint) {
this.constraint = constraint;
}
}

33
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java Normal file
View File

@ -0,0 +1,33 @@
package eu.dnetlib.openaire.community.selectioncriteria;
import java.io.Serializable;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.gson.Gson;
@JsonAutoDetect
public class SelectionCriteria implements Serializable {
/**
*
*/
private static final long serialVersionUID = 4303936216579280542L;
private List<Constraints> criteria;
public SelectionCriteria() {}
public List<Constraints> getCriteria() {
return criteria;
}
public void setCriteria(final List<Constraints> criteria) {
this.criteria = criteria;
}
public static SelectionCriteria fromJson(final String json) {
return new Gson().fromJson(json, SelectionCriteria.class);
}
}

67
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java Normal file
View File

@ -0,0 +1,67 @@
package eu.dnetlib.openaire.context;
import java.util.List;
import java.util.Map;
public class Category {
private String id;
private String label;
private boolean claim;
private Map<String, List<Param>> params;
private List<Concept> concepts;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public boolean isClaim() {
return claim;
}
public boolean hasConcepts() {
return getConcepts() != null && !getConcepts().isEmpty();
}
public Map<String, List<Param>> getParams() {
return params;
}
public List<Concept> getConcepts() {
return concepts;
}
public Category setId(final String id) {
this.id = id;
return this;
}
public Category setLabel(final String label) {
this.label = label;
return this;
}
public Category setClaim(final boolean claim) {
this.claim = claim;
return this;
}
public Category setParams(final Map<String, List<Param>> params) {
this.params = params;
return this;
}
public Category setConcepts(final List<Concept> concepts) {
this.concepts = concepts;
return this;
}
}

38
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java Normal file
View File

@ -0,0 +1,38 @@
package eu.dnetlib.openaire.context;
public class CategorySummary {
private String id;
private String label;
private boolean hasConcept;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public boolean isHasConcept() {
return hasConcept;
}
public CategorySummary setId(final String id) {
this.id = id;
return this;
}
public CategorySummary setLabel(final String label) {
this.label = label;
return this;
}
public CategorySummary setHasConcept(final boolean hasConcept) {
this.hasConcept = hasConcept;
return this;
}
}

67
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java Normal file
View File

@ -0,0 +1,67 @@
package eu.dnetlib.openaire.context;
import java.util.List;
import java.util.Map;
public class Concept {
private String id;
private String label;
private boolean claim;
private Map<String, List<Param>> params;
private List<Concept> concepts;
public boolean hasSubConcepts() {
return getConcepts() != null && !getConcepts().isEmpty();
}
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public boolean isClaim() {
return claim;
}
public Map<String, List<Param>> getParams() {
return params;
}
public List<Concept> getConcepts() {
return concepts;
}
public Concept setId(final String id) {
this.id = id;
return this;
}
public Concept setLabel(final String label) {
this.label = label;
return this;
}
public Concept setClaim(final boolean claim) {
this.claim = claim;
return this;
}
public Concept setParams(final Map<String, List<Param>> params) {
this.params = params;
return this;
}
public Concept setConcepts(final List<Concept> concepts) {
this.concepts = concepts;
return this;
}
}

51
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java Normal file
View File

@ -0,0 +1,51 @@
package eu.dnetlib.openaire.context;
import java.util.List;
public class ConceptSummary {
private String id;
private String label;
public boolean hasSubConcept;
private List<ConceptSummary> concepts;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public List<ConceptSummary> getConcepts() {
return concepts;
}
public ConceptSummary setId(final String id) {
this.id = id;
return this;
}
public ConceptSummary setLabel(final String label) {
this.label = label;
return this;
}
public boolean isHasSubConcept() {
return hasSubConcept;
}
public ConceptSummary setHasSubConcept(final boolean hasSubConcept) {
this.hasSubConcept = hasSubConcept;
return this;
}
public ConceptSummary setConcept(final List<ConceptSummary> concepts) {
this.concepts = concepts;
return this;
}
}

91
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java Normal file
View File

@ -0,0 +1,91 @@
package eu.dnetlib.openaire.context;
import java.util.Date;
import java.util.List;
import java.util.Map;
public class Context {
private String id;
private String label;
private String type;
private Date creationDate;
private Date lastUpdateDate;
private Map<String, List<Param>> params;
private Map<String, Category> categories;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public String getType() {
return type;
}
public Date getCreationDate() {
return creationDate;
}
public Map<String, List<Param>> getParams() {
return params;
}
public Map<String, Category> getCategories() {
return categories;
}
public Context setId(final String id) {
this.id = id;
return this;
}
public Context setLabel(final String label) {
this.label = label;
return this;
}
public Context setType(final String type) {
this.type = type;
return this;
}
public Context setCreationDate(final Date dateofcreation) {
this.creationDate = dateofcreation;
return this;
}
public Date getLastUpdateDate() {
return lastUpdateDate;
}
public Context setLastUpdateDate(Date lastUpdateDate) {
this.lastUpdateDate = lastUpdateDate;
return this;
}
public Context setParams(final Map<String, List<Param>> params) {
this.params = params;
return this;
}
public Context setCategories(final Map<String, Category> categories) {
this.categories = categories;
return this;
}
}

86
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java Normal file
View File

@ -0,0 +1,86 @@
package eu.dnetlib.openaire.context;
import java.util.List;
import java.util.Optional;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.*;
@RestController
@CrossOrigin(origins = { "*" })
@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
@io.swagger.annotations.Api(tags = "OpenAIRE Context API", description = "the OpenAIRE Context API")
public class ContextApiController {
@Autowired
private ContextApiCore contextApiCore;
@RequestMapping(value = "/contexts", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "list brief information about all the context profiles",
notes = "list brief information about all the context profiles.",
tags = { },
response = ContextSummary[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = ContextSummary[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
public List<ContextSummary> listContexts(@RequestParam(required = false, defaultValue = "") List<String> type) throws ContextException {
return contextApiCore.listContexts(type);
}
@RequestMapping(value = "/context/{contextId}", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "list the categories defined within a context",
notes = "list the categories defined within a context",
tags = { },
response = CategorySummary[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = CategorySummary[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
public List<CategorySummary> listCategories(
@PathVariable final String contextId,
@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
Boolean allFilter = Optional.ofNullable(all).orElse(false);
return contextApiCore.listCategories(contextId, allFilter);
}
@RequestMapping(value = "/context/category/{categoryId}", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "list the concepts defined within a category",
notes = "list the concepts defined within a category",
tags = { },
response = ConceptSummary[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
public List<ConceptSummary> listConcepts(
@PathVariable final String categoryId,
@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
Boolean allFilter = Optional.ofNullable(all).orElse(false);
return contextApiCore.listConcepts(categoryId, allFilter);
}
@RequestMapping(value = "/context/category/concept/{conceptId}", produces = { "application/json" }, method = RequestMethod.GET)
@ApiOperation(
value = "list the concepts defined within a category",
notes = "list the concepts defined within a category",
tags = { },
response = ConceptSummary[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) })
public List<ConceptSummary> listSubConcepts(
@PathVariable final String conceptId,
@RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException {
Boolean allFilter = Optional.ofNullable(all).orElse(false);
return contextApiCore.listSubConcepts(conceptId, allFilter);
}
}

113
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java Normal file
View File

@ -0,0 +1,113 @@
package eu.dnetlib.openaire.context;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.common.ISClient;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true")
public class ContextApiCore {
private static final String SEPARATOR = "::";
@Autowired
private ISClient isClient;
public List<ContextSummary> listContexts(final List<String> type) throws ContextException {
return getContextMap(type).values()
.stream()
.map(c -> new ContextSummary()
.setId(c.getId())
.setType(c.getType())
.setLabel(c.getLabel())
.setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : ""))
.collect(Collectors.toList());
}
public List<CategorySummary> listCategories(final String contextId, final Boolean all) throws ContextException {
final Stream<Category> categories = getContextMap().get(contextId).getCategories().values().stream();
return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim));
}
private List<CategorySummary> asCategorySummaries(final Stream<Category> categories) {
return categories
.map(c -> new CategorySummary()
.setId(c.getId())
.setLabel(c.getLabel())
.setHasConcept(c.hasConcepts()))
.collect(Collectors.toList());
}
public List<ConceptSummary> listConcepts(final String categoryId, final Boolean all) throws ContextException {
final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR);
final Stream<Concept> concepts = getContextMap().get(contextId)
.getCategories()
.get(categoryId)
.getConcepts()
.stream();
return all ? asConceptSummaries(concepts) : asConceptSummaries(concepts.filter(Concept::isClaim));
}
private List<ConceptSummary> asConceptSummaries(final Stream<Concept> concepts) {
return concepts
.map(c -> new ConceptSummary()
.setId(c.getId())
.setLabel(c.getLabel())
.setHasSubConcept(c.hasSubConcepts()))
.collect(Collectors.toList());
}
public List<ConceptSummary> listSubConcepts(final String conceptId, final Boolean all) throws ContextException {
final List<String> ids = Splitter.on(SEPARATOR).splitToList(conceptId);
if (ids.size() < 3) { throw new ContextException(""); }
final String contextId = ids.get(0);
final String categoryId = contextId + SEPARATOR + ids.get(1);
final Stream<Concept> concepts = getContextMap().get(contextId)
.getCategories()
.get(categoryId)
.getConcepts()
.stream()
.filter(c -> conceptId.equals(c.getId()));
return all ? mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) : mapConcepts(concepts.collect(Collectors.toList()));
}
private List<ConceptSummary> mapConcepts(final List<Concept> concepts) {
if (concepts == null || concepts.isEmpty()) { return null; }
return concepts.stream()
.map(c -> new ConceptSummary()
.setId(c.getId())
.setLabel(c.getLabel())
.setHasSubConcept(c.hasSubConcepts())
.setConcept(mapConcepts(c.getConcepts())))
.collect(Collectors.toList());
}
private Map<String, Context> getContextMap() throws ContextException {
return getContextMap(Lists.newArrayList());
}
private Map<String, Context> getContextMap(final List<String> type) throws ContextException {
try {
return isClient.getContextMap(type);
} catch (final IOException e) {
throw new ContextException(e);
}
}
}

25
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.context;
import java.io.IOException;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public class ContextException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5489369676370127052L;
public ContextException(final String message) {
super(message);
}
public ContextException(final IOException e) {
super(e);
}
}

113
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java Normal file
View File

@ -0,0 +1,113 @@
package eu.dnetlib.openaire.context;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.stream.Collectors;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.dom4j.Node;
import com.google.common.base.Functions;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.funders.domain.FunderDetails;
public class ContextMappingUtils {
private static final List<String> DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00");
public static Context parseContext(final String s, final Queue<Throwable> errors) {
try {
final Document doc = DocumentHelper.parseText(s);
final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context");
final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()");
final Context c = new Context()
.setId(eContext.attributeValue("id"))
.setLabel(eContext.attributeValue("label"))
.setType(eContext.attributeValue("type"))
.setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value")))
.setParams(parseParams(eContext))
.setCategories(parseCategories(eContext));
// the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check.
if (StringUtils.isNotBlank(creationDate)) {
c.setCreationDate(asDate(creationDate));
}
return c;
} catch (final DocumentException e) {
errors.add(e);
return new Context();
}
}
private static Date asDate(final String s) {
for (final String pattern : DATE_PATTERN) {
try {
return DateUtils.parseDate(s, pattern);
} catch (final ParseException e) {}
}
return null;
}
private static Map<String, Category> parseCategories(final Element eContext) {
final List<Node> eCategory = eContext.selectNodes("//category");
return eCategory.stream()
.map(n -> (Element) n)
.map(eCat -> new Category()
.setClaim(getClaim(eCat))
.setId(eCat.attributeValue("id"))
.setLabel(eCat.attributeValue("label"))
.setParams(parseParams(eCat))
.setConcepts(parseConcepts(eCat)))
.collect(Collectors.toMap(Category::getId, Functions.identity()));
}
private static List<Concept> parseConcepts(final Element eCategory) {
final List<Node> eConcepts = eCategory.selectNodes("./concept");
return eConcepts.stream()
.map(n -> (Element) n)
.map(eCon -> new Concept()
.setClaim(getClaim(eCon))
.setId(eCon.attributeValue("id"))
.setLabel(eCon.attributeValue("label"))
.setParams(parseParams(eCon))
.setConcepts(parseConcepts(eCon)))
.collect(Collectors.toList());
}
private static Boolean getClaim(final Element eCon) {
final String claim = eCon.attributeValue("claim");
return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
}
private static Map<String, List<Param>> parseParams(final Element e) {
final List<Node> params = e.selectNodes("./param");
return params.stream()
.map(n -> (Element) n)
.map(p -> new Param()
.setName(p.attributeValue("name"))
.setValue(p.getTextTrim()))
.collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
final List<Param> p = new ArrayList<>(p1);
p.addAll(p2);
return p;
}));
}
public static FunderDetails asFunderDetails(final Context c) {
return new FunderDetails()
.setId(c.getId())
.setName(c.getLabel())
.setShortname(c.getId());
}
}

23
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java Normal file
View File

@ -0,0 +1,23 @@
package eu.dnetlib.openaire.context;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseBody
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public class ContextNotFoundException extends Exception {
/**
*
*/
private static final long serialVersionUID = -2026506752817353752L;
public ContextNotFoundException(final String msg) {
super(msg);
}
public ContextNotFoundException(final Exception e) {
super(e);
}
}

49
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java Normal file
View File

@ -0,0 +1,49 @@
package eu.dnetlib.openaire.context;
public class ContextSummary {
private String id;
private String label;
private String type;
private String status;
public String getId() {
return id;
}
public String getLabel() {
return label;
}
public String getType() {
return type;
}
public String getStatus() {
return status;
}
public ContextSummary setId(final String id) {
this.id = id;
return this;
}
public ContextSummary setLabel(final String label) {
this.label = label;
return this;
}
public ContextSummary setType(final String type) {
this.type = type;
return this;
}
public ContextSummary setStatus(final String status) {
this.status = status;
return this;
}
}

27
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java Normal file
View File

@ -0,0 +1,27 @@
package eu.dnetlib.openaire.context;
public class Param {
private String name;
private String value;
public String getName() {
return name;
}
public String getValue() {
return value;
}
public Param setName(final String name) {
this.name = name;
return this;
}
public Param setValue(final String value) {
this.value = value;
return this;
}
}

407
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java Executable file
View File

@ -0,0 +1,407 @@
package eu.dnetlib.openaire.dsm;
import static eu.dnetlib.openaire.common.ExporterConstants.API;
import static eu.dnetlib.openaire.common.ExporterConstants.DS;
import static eu.dnetlib.openaire.common.ExporterConstants.M;
import static eu.dnetlib.openaire.common.ExporterConstants.R;
import static eu.dnetlib.openaire.common.ExporterConstants.W;
import java.util.List;
import javax.validation.Valid;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.http.HttpStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.common.OperationManager;
import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.Response;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.vocabularies.Country;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
@RestController
@CrossOrigin(origins = {
"*"
})
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
@io.swagger.annotations.Api(tags = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API")
public class DsmApiController extends AbstractExporterController {
@Autowired
private DsmCore dsmCore;
@RequestMapping(value = "/ds/countries", produces = {
"application/json"
}, method = RequestMethod.GET)
@ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = {
DS, R
}, response = Country[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = Country[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public List<Country> listCountries() throws DsmException {
return dsmCore.listCountries();
}
@RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
@ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = {
DS, R
}, response = DatasourceDetailResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public DatasourceDetailResponse searchDsDetails(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size);
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = {
"application/json"
}, method = RequestMethod.GET)
@ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = {
DS, R
}, response = AggregationHistoryResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId);
return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp);
}
@RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
@ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = {
DS, R
}, response = DatasourceSnippetResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public DatasourceSnippetResponse searchSnippet(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size);
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
@ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = {
DS,
R
}, response = DatasourceSnippetResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public DatasourceSnippetResponse searchRegistered(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size);
return prepareResponse(page, size, stop, rsp);
}
@RequestMapping(value = "/ds/recentregistered/{size}", produces = {
"application/json"
}, method = RequestMethod.GET)
@ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = {
DS,
R
}, response = SimpleResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public SimpleResponse<?> recentRegistered(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<?> rsp = dsmCore.searchRecentRegistered(size);
return prepareResponse(1, size, stop, rsp);
}
@RequestMapping(value = "/ds/countregistered", produces = {
"application/json"
}, method = RequestMethod.GET)
@ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = {
DS,
R
}, response = Long.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = Long.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public Long countRegistered(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
return dsmCore.countRegisteredAfter(fromDate, typologyFilter);
}
@RequestMapping(value = "/ds/api/{dsId}", produces = {
"application/json"
}, method = RequestMethod.GET)
@ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = {
API,
R
}, response = ApiDetailsResponse.class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public ApiDetailsResponse getApi(
@PathVariable final String dsId) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final ApiDetailsResponse rsp = dsmCore.getApis(dsId);
return prepareResponse(0, rsp.getApi().size(), stop, rsp);
}
@RequestMapping(value = "/api/baseurl/{page}/{size}", produces = {
"application/json"
}, method = RequestMethod.POST)
@ApiOperation(value = "search for the list of base URLs of Datasource APIs managed by a user", notes = "Returns the list of base URLs of Datasource APIs managed by a user", tags = {
DS, API, R
}, response = String[].class)
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = String[].class),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public List<String> searchBaseUrls(
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
return dsmCore.findBaseURLs(requestFilter, page, size);
}
@RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE)
@ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = {
API, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class),
@ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class),
@ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class)
})
public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException {
dsmCore.deleteApi(apiId);
}
@RequestMapping(value = "/ds/manage", method = RequestMethod.POST)
@ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = {
DS, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void setManaged(
@RequestParam final String id,
@RequestParam final boolean managed) throws DsmException {
dsmCore.setManaged(id, managed);
}
@RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET)
@ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = {
DS, R
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public boolean isManaged(@PathVariable final String id) throws DsmException {
return dsmCore.isManaged(id);
}
@RequestMapping(value = "/ds/add", method = RequestMethod.POST)
@ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = {
DS, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class),
@ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class)
})
public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException {
if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API
throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", datasource.getId()));
}
dsmCore.save(datasource);
}
@RequestMapping(value = "/ds/update", method = RequestMethod.POST)
@ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = {
DS, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void updateDatasource(
@RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException {
dsmCore.updateDatasource(ds);
}
@RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST)
@ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = {
API, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void updateBaseUrl(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String baseUrl) throws DsmException {
dsmCore.updateApiBaseurl(dsId, apiId, baseUrl);
}
@RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST)
@ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = {
API, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void updateCompliance(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String compliance,
@RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException {
dsmCore.updateApiCompatibility(dsId, apiId, compliance, override);
}
@RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST)
@ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = {
API, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void updateOaiSetl(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String oaiSet) throws DsmException {
dsmCore.updateApiOaiSet(dsId, apiId, oaiSet);
}
@RequestMapping(value = "/ds/api/add", method = RequestMethod.POST)
@ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = {
API, W
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void addApi(@RequestBody final ApiDetails api) throws DsmException {
if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); }
dsmCore.addApi(api);
}
// MANAGEMENT
@Autowired
private OperationManager operationManager;
@RequestMapping(value = "/dsm/ops", method = RequestMethod.GET)
@ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = {
R, M
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public int getOps() throws DsmException {
return operationManager.getOpSize();
}
@RequestMapping(value = "/dsm/killops", method = RequestMethod.POST)
@ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = {
W, M
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public int killOps() throws DsmException {
return operationManager.dropAll();
}
@RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST)
@ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = {
W, M
})
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK"),
@ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class)
})
public void dropCache() throws DsmException {
dsmCore.dropCaches();
}
// HELPERS
private <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
rsp.getHeader()
.setTime(stopWatch.getTime())
.setPage(page)
.setSize(size);
return rsp;
}
}

296
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java Normal file
View File

@ -0,0 +1,296 @@
package eu.dnetlib.openaire.dsm;
import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDbEntry;
import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDetails;
import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.copyNonNullProperties;
import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.createId;
import java.nio.charset.Charset;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.community.CommunityClient;
import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
import eu.dnetlib.openaire.dsm.dao.VocabularyClient;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry;
import eu.dnetlib.openaire.vocabularies.Country;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public class DsmCore {
private static final Log log = LogFactory.getLog(DsmCore.class);
@Autowired
private MongoLoggerClient mongoLoggerClient;
@Autowired
private ISClient isClient;
@Autowired
private VocabularyClient vocabularyClient;
@Autowired
private DatasourceDao<DatasourceDbEntry, ApiDbEntry> dsDao;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private CommunityClient communityClient;
public List<Country> listCountries() throws DsmException {
try {
return dsDao.listCountries();
} catch (final Throwable e) {
log.error("error listing countries", e);
throw e;
}
}
public DatasourceDetailResponse searchDsDetails(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
try {
final Page<DatasourceDbEntry> dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.detailsResponse(dsPage.map(d -> asDetails(d)).getContent(), dsPage.getTotalElements());
} catch (final Throwable e) {
log.error("error searching datasources", e);
throw e;
}
}
public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
try {
final Page<DatasourceDbEntry> dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
} catch (final Throwable e) {
log.error("error searching datasources", e);
throw e;
}
}
public DatasourceSnippetResponse searchRegistered(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
try {
final Page<DatasourceDbEntry> dsPage = dsDao.searchRegistered(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
} catch (final Throwable e) {
log.error("error searching datasources", e);
throw e;
}
}
public List<String> findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
try {
return dsDao.findApiBaseURLs(requestFilter, page, size);
} catch (final Throwable e) {
log.error("error searching datasource base urls", e);
throw e;
}
}
public ApiDetailsResponse getApis(final String dsId) throws DsmException {
try {
final String eoscType = dsDao.getDs(dsId).getEoscDatasourceType();
final List<? extends ApiDbEntry> apis = dsDao.getApis(dsId);
final List<ApiDetails> api = apis.stream()
.map(DsmMappingUtils::asDetails)
.map(a -> a.setEoscDatasourceType(eoscType))
.collect(Collectors.toList());
return ResponseUtils.apiResponse(api, api.size());
} catch (final Throwable e) {
log.error(String.format("error searching datasource api %s", dsId), e);
throw e;
}
}
public void setManaged(final String dsId, final boolean managed) throws DsmException {
log.info(String.format("updated ds '%s' managed with '%s'", dsId, managed));
dsDao.setManaged(dsId, managed);
}
public boolean isManaged(final String dsId) throws DsmException {
return dsDao.isManaged(dsId);
}
public boolean exist(final DatasourceDetails d) throws DsmException {
return dsDao.existDs(d.getId());
}
public void save(final DatasourceDetails d) throws DsmException {
try {
dsDao.saveDs(asDbEntry(d));
} catch (final Throwable e) {
log.error(ExceptionUtils.getStackTrace(e));
throw e;
}
}
public void updateDatasource(final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException {
try {
// initialize with current values from DB
final DatasourceDbEntry ds = dsDao.getDs(d.getId());
if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); }
final DatasourceDbEntry update = asDbEntry(d);
if (d.getIdentities() != null) {
final Set<IdentityDbEntry> identities = new HashSet<>(
Stream.of(update.getIdentities(), ds.getIdentities())
.flatMap(Collection::stream)
.collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1))
.values());
copyNonNullProperties(update, ds);
ds.setIdentities(identities);
} else {
copyNonNullProperties(update, ds);
}
dsDao.saveDs(ds);
} catch (final Throwable e) {
log.error(ExceptionUtils.getStackTrace(e));
throw e;
}
}
// TODO remove if unused
public void deleteDs(final String dsId) throws DsmException {
log.info(String.format("deleted datasource '%s'", dsId));
dsDao.deleteDs(dsId);
}
// API
public void updateApiOaiSet(final String dsId, final String apiId, final String oaiSet) throws DsmException {
dsDao.upsertApiOaiSet(apiId, oaiSet);
}
public void updateApiBaseurl(final String dsId, final String apiId, final String baseUrl) throws DsmException {
log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl));
dsDao.updateApiBaseUrl(apiId, baseUrl);
}
public void updateApiCompatibility(final String dsId, final String apiId, final String compliance, final boolean override) throws DsmException {
log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance));
dsDao.updateCompliance(null, apiId, compliance, override);
}
public void addApi(final ApiDetails api) throws DsmException {
if (StringUtils.isBlank(api.getId())) {
api.setId(createId(api));
log.info(String.format("missing api id, created '%s'", api.getId()));
}
dsDao.addApi(asDbEntry(api));
}
public void deleteApi(final String apiId) throws DsmForbiddenException, DsmNotFoundException {
// TODO handle the api removal in case of associated workflows.
dsDao.deleteApi(null, apiId);
}
public void dropCaches() {
mongoLoggerClient.dropCache();
isClient.dropCache();
vocabularyClient.dropCache();
communityClient.dropCache();
}
// HELPERS //////////////
public SimpleResponse<?> searchRecentRegistered(final int size) throws Throwable {
try {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset());
final List<RegisteredDatasourceInfo> list = jdbcTemplate.query(sql, BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw e;
}
}
public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%");
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw e;
}
}
public AggregationHistoryResponse aggregationhistory(final String dsId) throws DsmException {
final List<AggregationInfo> history = mongoLoggerClient.getAggregationHistory(dsId);
final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history);
rsp.setHeader(ResponseUtils.header(history.size()));
return rsp;
}
}

53
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java Normal file
View File

@ -0,0 +1,53 @@
package eu.dnetlib.openaire.dsm.dao;
import java.util.List;
import javax.transaction.Transactional;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
/**
* Created by claudio on 15/06/2017.
*/
@Repository
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public interface ApiDbEntryRepository extends JpaRepository<ApiDbEntry, String> {
@Query("select a from #{#entityName} a where a.datasource = ?1")
List<ApiDbEntry> findByDatasource(String dsId);
@Modifying
@Transactional
@Query("update #{#entityName} a set a.baseurl = ?2 where a.id = ?1")
void setBaseurl(String id, String baseurl);
@Modifying
@Transactional
@Query("update #{#entityName} a set a.compatibility = ?2 where a.id = ?1")
void updateCompatibility(String apiId, String compatibility);
@Modifying
@Transactional
@Query("update #{#entityName} a set a.compatibilityOverride = ?2 where a.id = ?1")
void updateCompatibilityOverride(String apiId, String compatibility);
@Modifying
@Transactional
@Query(value = "update dsm_apiparams ap set value = ?2 where ap.param = 'set' and ap.api = ?1", nativeQuery = true)
void updateOaiSet(String apiId, String oaiSet);
@Modifying
@Transactional
@Query(value = "insert into dsm_apiparams(api, param, value, _dnet_resource_identifier_) values(?1, ?2, ?3, ?1||'@@'||?2)", nativeQuery = true)
void addApiParam(String apiId, String param, String value);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.removable = ?2 where d.datasource = ?1")
void setRemovable(String id, boolean removable);
}

15
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java Normal file
View File

@ -0,0 +1,15 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.openaire.dsm.domain.db.CountryTerm;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
/**
* Created by claudio on 19/04/2017.
*/
@Repository
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public interface CountryTermRepository extends JpaRepository<CountryTerm, String> {
}

14
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.stereotype.Repository;
@Repository
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public interface DatasourceApiDbEntryRepository extends JpaRepository<DatasourceApiDbEntry, String>, JpaSpecificationExecutor<DatasourceApiDbEntry> {
}

60
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java Normal file
View File

@ -0,0 +1,60 @@
package eu.dnetlib.openaire.dsm.dao;
import java.util.List;
import org.springframework.data.domain.Page;
import eu.dnetlib.enabling.datasources.common.Api;
import eu.dnetlib.enabling.datasources.common.Datasource;
import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.vocabularies.Country;
public interface DatasourceDao<DS extends Datasource<?, ?, ?>, API extends Api<?>> extends DatasourceManagerCommon<DS, API> {
// DATASOURCE
List<Country> listCountries() throws DsmException;
boolean existDs(final String dsId) throws DsmException;
Page<DS> search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException;
Page<DS> searchRegistered(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException;
void updateName(String dsId, String officialname, String englishname) throws DsmException;
void updateLogoUrl(String dsId, String logourl) throws DsmException;
void updateCoordinates(String dsId, Double latitude, Double longitude) throws DsmException;
void updateTimezone(String dsId, String timezone) throws DsmException;
void updateEoscDatasourceType(String dsId, String timezone) throws DsmException;
void updateRegisteringUser(String dsId, String registeredBy) throws DsmException;
void updatePlatform(String dsId, String platform) throws DsmException;
// API
List<String> findApiBaseURLs(RequestFilter requestFilter, int page, int size) throws DsmException;
/**
* Insert the oai set in case it does not exists, updates it otherwise
*
* @param apiId
* @param oaiSet
* @return true in case of insert, false in case of update
* @throws DsmException
*/
boolean upsertApiOaiSet(String apiId, String oaiSet) throws DsmException;
void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException;
@Override
void addApi(final API api) throws DsmException;
}

274
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java Normal file
View File

@ -0,0 +1,274 @@
package eu.dnetlib.openaire.dsm.dao;
import static eu.dnetlib.openaire.common.ExporterConstants.OAI;
import static eu.dnetlib.openaire.common.ExporterConstants.SET;
import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec;
import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec;
import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec;
import java.sql.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.EntityNotFoundException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpStatus;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Lists;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
import eu.dnetlib.openaire.vocabularies.Country;
import eu.dnetlib.openaire.vocabularies.Vocabulary;
/**
* Created by claudio on 20/10/2016.
*/
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public class DatasourceDaoImpl implements DatasourceDao<DatasourceDbEntry, ApiDbEntry> {
private static final Log log = LogFactory.getLog(DatasourceDao.class);
@Autowired
private DnetOpenaireExporterProperties config;
@Autowired
private CountryTermRepository countryTermRepository;
@Autowired
private DatasourceDbEntryRepository dsRepository;
@Autowired
private ApiDbEntryRepository apiRepository;
@Autowired
private DatasourceApiDbEntryRepository dsApiRepository;
@Autowired
private VocabularyClient vocabularyClient;
@Override
public List<Country> listCountries() throws DsmException {
final List<Country> countries = Lists.newArrayList();
final Vocabulary v = vocabularyClient.getCountries();
countries.addAll(countryTermRepository.findAll()
.stream()
.filter(Objects::nonNull)
.map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm())))
.collect(Collectors.toList()));
return countries;
}
@Override
public Page<DatasourceDbEntry> search(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<DatasourceDbEntry> spec = dsSpec(requestSortBy, order, requestFilter);
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
@Override
public Page<DatasourceDbEntry> searchRegistered(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<DatasourceDbEntry> spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec());
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
@Override
public DatasourceDbEntry getDs(final String dsId) throws DsmException {
return dsRepository.findById(dsId).orElseThrow(() -> new DsmException("Datasource not found. ID: " + dsId));
}
@Override
public DatasourceDbEntry getDsByNsPrefix(final String prefix) throws DsmException {
return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix));
}
@Override
public void setManaged(final String id, final boolean managed) {
log.info(String.format("setting managed = '%s' for ds '%s'", managed, id));
dsRepository.setManaged(id, managed);
apiRepository.setRemovable(id, true);
}
@Override
public boolean isManaged(final String id) {
return dsRepository.isManaged(id);
}
@Override
public void updateCompliance(final String dsId, final String apiId, final String compliance, final boolean override) {
log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId));
apiRepository.updateCompatibility(apiId, compliance);
}
@Override
public List<ApiDbEntry> getApis(final String dsId) {
return apiRepository.findByDatasource(dsId);
}
@Override
public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException {
final ApiDbEntry api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
try {
if (!api.getRemovable()) { throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); }
apiRepository.deleteById(apiId);
log.info(String.format("deleted api '%s'", apiId));
} catch (final EntityNotFoundException e) {
throw new DsmNotFoundException(HttpStatus.SC_NOT_FOUND, "api not found");
}
}
@Override
public void addApi(final ApiDbEntry api) {
apiRepository.save(api);
}
@Override
public boolean existDs(final String dsId) throws DsmException {
return dsRepository.existsById(dsId);
}
@Override
public void saveDs(final DatasourceDbEntry d) {
log.info(String.format("saving datasource '%s'", d.getId()));
final DatasourceDbEntry datasource = dsRepository.save(d);
log.info(String.format("saved datasource '%s'", datasource.getId()));
ensureRegistrationDate(d.getId());
}
@Override
public void deleteDs(final String dsId) {
dsRepository.deleteById(dsId);
log.info(String.format("deleted datasource '%s'", dsId));
}
@Override
public void updateName(final String dsId, final String officialname, final String englishname) {
// TODO what if one of the two names is null or empty?
dsRepository.setDatasourcename(dsId, officialname, englishname);
}
@Override
public void updateLogoUrl(final String dsId, final String logourl) throws DsmException {
dsRepository.setLogoUrl(dsId, logourl);
}
@Override
public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) {
dsRepository.setCoordinates(dsId, latitude, longitude);
}
@Override
public void updateApiBaseUrl(final String apiId, final String baseurl) {
apiRepository.setBaseurl(apiId, baseurl);
}
@Override
@Transactional
public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException {
final ApiDbEntry api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
if (OAI.equalsIgnoreCase(api.getProtocol())) {
final Set<ApiParamDbEntry> apiParams = api.getApiParams();
if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) {
apiRepository.addApiParam(apiId, SET, oaiSet);
log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet));
return true;
} else {
apiRepository.updateOaiSet(apiId, oaiSet);
log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet));
return false;
}
} else {
throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol()));
}
}
@Override
public List<String> findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
final PageRequest pageable = PageRequest.of(page, size);
final Specification<DatasourceApiDbEntry> spec = apiSpec(requestFilter);
final Set<String> set = dsApiRepository.findAll(spec, pageable)
.getContent()
.stream()
.map(DatasourceApiDbEntry::getBaseurl)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toCollection(HashSet::new));
return Lists.newArrayList(set);
}
@Override
public void updateTimezone(final String dsId, final String timezone) {
dsRepository.setTimezone(dsId, timezone);
}
@Override
public void updateEoscDatasourceType(final String dsId, final String type) throws DsmException {
final Vocabulary typologies = vocabularyClient.getDatasourceTypologies();
if (!typologies.hasCode(type)) {
throw new DsmException(
HttpStatus.SC_BAD_REQUEST,
String.format("invalid datasource type '%s', provide one according to vocabulary %s", type, config.getVocabularies()
.getDatasourceTypologiesEndpoint()));
}
dsRepository.setEoscDatasourceType(dsId, type);
}
@Override
public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException {
ensureRegistrationDate(dsId);
dsRepository.setRegisteringUser(dsId, registeredBy);
}
@Override
public void updatePlatform(final String dsId, final String platform) throws DsmException {
dsRepository.setPlatform(dsId, platform);
}
// HELPER
private void ensureRegistrationDate(final String dsId) {
if (!dsRepository.hasRegistrationdate(dsId)) {
log.info("setting registration date for datasource: " + dsId);
dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis()));
}
}
}

77
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java Normal file
View File

@ -0,0 +1,77 @@
package eu.dnetlib.openaire.dsm.dao;
import java.sql.Date;
import java.util.Optional;
import javax.transaction.Transactional;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
/**
* Created by claudio on 12/04/2017.
*/
@Repository
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public interface DatasourceDbEntryRepository extends JpaRepository<DatasourceDbEntry, String>, JpaSpecificationExecutor<DatasourceDbEntry> {
Optional<DatasourceDbEntry> findByNamespaceprefix(String namespaceprefix);
@Query("select d.managed from #{#entityName} d where d.id = ?1")
boolean isManaged(String id);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.managed = ?2 where d.id = ?1")
void setManaged(String id, boolean managed);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.officialname = ?2, d.englishname = ?3 where d.id = ?1")
void setDatasourcename(String id, String officialname, String englishname);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.logourl = ?2 where d.id = ?1")
void setLogoUrl(String dsId, String logourl);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.latitude = ?2, d.longitude = ?3 where d.id = ?1")
void setCoordinates(String dsId, Double latitude, Double longitude);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.timezone = ?2 where d.id = ?1")
void setTimezone(String dsId, String timezone);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.eoscDatasourceType = ?2 where d.id = ?1")
void setEoscDatasourceType(String dsId, String type);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.registeredby = ?2 where d.id = ?1")
void setRegisteringUser(String id, String registeredby);
@Query("select case when registrationdate <> null then true else false end as hasregistrationdate from #{#entityName} where id = ?1")
Boolean hasRegistrationdate(String id);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1")
void setRegistrationDate(String id, Date registrationdate);
@Modifying
@Transactional
@Query("update #{#entityName} d set d.platform = ?2 where d.id = ?1")
void setPlatform(String id, String platform);
}

17
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java Normal file
View File

@ -0,0 +1,17 @@
package eu.dnetlib.openaire.dsm.dao;
import java.util.Queue;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo;
public interface DatasourceIndexClient {
IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue<Throwable> errors) throws DsmException;
String getLastIndexingDate(final IndexDsInfo info) throws DsmException;
}

207
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java Normal file
View File

@ -0,0 +1,207 @@
package eu.dnetlib.openaire.dsm.dao;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.*;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.*;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.miscutils.functional.hash.Hashing;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
/**
* Created by claudio on 20/10/2016.
*/
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public class DatasourceIndexClientImpl implements DatasourceIndexClient {
private static final Log log = LogFactory.getLog(DatasourceIndexClientImpl.class);
public static final String SEPARATOR = "::";
public static final String DSVERSION = "__dsversion";
@Autowired
private DnetOpenaireExporterProperties config;
private ListeningExecutorService executor;
private static final Map<String, CloudSolrClient> indexClientMap = new ConcurrentHashMap<>();
@PostConstruct
public void init() {
executor = MoreExecutors.listeningDecorator(
new ScheduledThreadPoolExecutor(5,
new ThreadFactoryBuilder().setNameFormat("datasource-index-client-%d").build()));
}
@PreDestroy
public void tearDown() {
indexClientMap.forEach((name, client) -> {
try {
client.close();
} catch (IOException e) {
log.warn(String.format("unable to gracefully shutdown client for index %s", name));
}
});
}
@Override
public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue<Throwable> errors) throws DsmException {
try {
final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR));
final CloudSolrClient indexClient = getIndexClient(info);
final CountDownLatch latch = new CountDownLatch(2);
final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo();
Futures.addCallback(
executor.submit(() -> setDateAndTotal(dsId, collectedFrom, indexClient)),
new FutureCallback<IndexRecordsInfo>() {
@Override
public void onSuccess(final IndexRecordsInfo info) {
indexRecordInfo
.setTotal(info.getTotal())
.setDate(info.getDate());
latch.countDown();
}
@Override
public void onFailure(final Throwable e) {
errors.offer(e);
latch.countDown();
}
}, executor);
Futures.addCallback(
executor.submit(() -> setFunded(dsId, collectedFrom, indexClient)),
new FutureCallback<Long>() {
@Override
public void onSuccess(final Long numFound) {
indexRecordInfo.setFunded(numFound);
latch.countDown();
}
@Override
public void onFailure(final Throwable e) {
errors.offer(e);
latch.countDown();
}
}, executor);
waitLatch(latch, errors, config.getRequestTimeout());
return indexRecordInfo;
} catch (final Throwable e) {
throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("error reading index info", dsId), e);
}
}
@Override
public String getLastIndexingDate(final IndexDsInfo info) throws DsmException {
try {
final SolrQuery query = new SolrQuery("oaftype:datasource").setRows(1);
final QueryResponse rsp = getIndexClient(info).query(query);
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);
final String dsversion = doc.get("__dsversion").toString();
return StringUtils.substringBefore(dsversion, "T");
} catch (SolrServerException | IOException e) {
throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e);
}
}
private Long setFunded(
final String dsId,
final String collectedFrom,
final CloudSolrClient indexClient) throws DsmException {
final String query =
String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom);
log.debug(String.format("query: %s", query));
try {
return indexClient.query(new SolrQuery(query).setRows(0)).getResults().getNumFound();
} catch (Throwable e) {
throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for funded results '%s'", dsId), e);
}
}
private IndexRecordsInfo setDateAndTotal(
final String dsId,
final String collectedFrom,
final CloudSolrClient indexClient) throws DsmException {
try {
final String query = String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom);
log.debug(String.format("query: %s", query));
final QueryResponse rsp = indexClient.query(new SolrQuery(query).setRows(1));
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument());
if (log.isDebugEnabled()) {
log.debug(String.format("got document %s", doc.get("__indexrecordidentifier")));
}
// if (doc.isEmpty()) {
// throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching
// query: %s", queryTotal));
// }
return new IndexRecordsInfo()
.setDate(getDate(doc))
.setTotal(rsp.getResults().getNumFound());
} catch (Throwable e) {
throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for date and total '%s'", dsId), e);
}
}
@SuppressWarnings("unchecked")
private String getDate(final SolrDocument doc) throws DsmException {
final List<Date> dsversion = (List<Date>) doc.get(DSVERSION);
if (dsversion == null || dsversion.isEmpty()) { throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(),
String.format("cannot find %s in matched solr document", DSVERSION)); }
final Date date = Iterables.getLast(dsversion);
return DateFormatUtils.format(date, DsmMappingUtils.DATE_FORMAT);
}
private synchronized CloudSolrClient getIndexClient(final IndexDsInfo info) {
if (!indexClientMap.containsKey(info.getColl())) {
final CloudSolrClient client = new Builder(Lists.newArrayList(info.getIndexBaseUrl())).build();
client.setDefaultCollection(info.getColl());
indexClientMap.put(info.getColl(), client);
}
return indexClientMap.get(info.getColl());
}
private void waitLatch(final CountDownLatch latch, final Queue<Throwable> errors, final int waitSeconds) {
try {
if (!latch.await(waitSeconds, TimeUnit.SECONDS)) {
errors.offer(new TimeoutException("Waiting for requests to complete has timed out."));
}
} catch (final InterruptedException e) {
errors.offer(e);
}
}
}

141
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java Normal file
View File

@ -0,0 +1,141 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.enabling.datasources.common.DsmRuntimeException;
import eu.dnetlib.openaire.dsm.domain.FilterName;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.data.jpa.domain.Specification;
import javax.persistence.criteria.*;
import java.util.List;
import java.util.Map.Entry;
public class DatasourceSpecs {
private static final Log log = LogFactory.getLog(DatasourceSpecs.class);
public static final String WILDCARD = "%";
public static Specification<DatasourceDbEntry> dsRegisteredbyNotNullSpec() {
return (ds, query, cb) -> cb.and(
cb.isNull(ds.get(FilterName.registeredby.name())).not(),
cb.isNull(ds.get("registrationdate")).not());
}
public static Specification<DatasourceDbEntry> dsSpec(final RequestSort requestSortBy, final RequestSortOrder order, final RequestFilter requestFilter) {
log.debug(String.format("RequestFilter:'%s', RequestSort:'%s', RequestSortOrder:'%s'", requestFilter, requestSortBy, order));
return (ds, query, cb) -> {
final Predicate p = cb.conjunction();
if (requestFilter != null) {
final List<Expression<Boolean>> expressions = p.getExpressions();
requestFilter.entrySet().stream()
.forEach(e -> {
switch (FilterName.type(e.getKey())) {
case exact:
expressions.add(exactSearch(ds, cb, e));
break;
case search:
expressions.add(likeSearch(ds, cb, e));
break;
case searchOrgs:
// search by case insensitive organization's country
expressions.add(
cb.equal(
cb.lower(
ds.join("organizations").get(FilterName.country.name())),
getValue(e)));
break;
}
});
}
if (requestSortBy != null) {
if (order != null) {
final Path<Object> orderField = ds.get(requestSortBy.name());
switch (order) {
case ASCENDING:
query.orderBy(cb.asc(orderField));
break;
case DESCENDING:
query.orderBy(cb.desc(orderField));
break;
}
}
}
query.distinct(true);
return p;
};
}
public static Specification<DatasourceApiDbEntry> apiSpec(final RequestFilter requestFilter) {
log.debug(String.format("RequestFilter:'%s'", requestFilter));
return (api, query, cb) -> {
final Predicate p = cb.conjunction();
if (requestFilter != null) {
final List<Expression<Boolean>> expressions = p.getExpressions();
requestFilter.entrySet().stream()
.forEach(e -> {
switch (FilterName.type(e.getKey())) {
case exact:
expressions.add(exactSearch(api, cb, e));
break;
case search:
expressions.add(likeSearch(api, cb, e));
break;
case searchOrgs:
throw new DsmRuntimeException("not implemented");
}
});
}
query.distinct(true);
return p;
};
}
// HELPERS
// substring, case insensitive, like based search
private static Predicate likeSearch(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
return cb.like(
cb.lower(
r.get(e.getKey().name())),
WILDCARD + getValue(e) + WILDCARD);
}
// search by ID, managed. exact match
private static Predicate exactSearch(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
return cb.equal(r.get(e.getKey().name()), getValue(e));
}
private static Object getValue(final Entry<FilterName, Object> e) {
if (e.getValue() instanceof String) {
final String s = ((String) e.getValue());
if (!e.getKey().equals(FilterName.country)) {
Boolean b = BooleanUtils.toBooleanObject(s);
if (b != null) {
return b;
}
}
return e.getKey().equals(FilterName.id) ? s : StringUtils.lowerCase(s);
}
if (e.getValue() instanceof Boolean) {
return e.getValue();
}
return e.getValue();
}
}

14
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.DsmException;
import java.util.List;
public interface MongoLoggerClient {
List<AggregationInfo> getAggregationHistory(final String dsId) throws DsmException;
void dropCache();
}

234
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java Normal file
View File

@ -0,0 +1,234 @@
package eu.dnetlib.openaire.dsm.dao;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.cache.*;
import com.google.common.primitives.Ints;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.AggregationStage;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.miscutils.datetime.DateUtils;
import eu.dnetlib.openaire.common.Utils;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
import eu.dnetlib.openaire.dsm.domain.CollectionMode;
import eu.dnetlib.openaire.dsm.domain.TransformationInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpStatus;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import static com.mongodb.client.model.Filters.*;
import static com.mongodb.client.model.Projections.fields;
/**
* Created by claudio on 20/10/2016.
*/
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public class MongoLoggerClientImpl implements MongoLoggerClient {
private static final Log log = LogFactory.getLog(MongoLoggerClientImpl.class);
@Autowired
private MongoClient datasourcePublisherMongoClient;
@Autowired
private DnetOpenaireExporterProperties config;
private final static String LOADTIME = "loadtime";
private final LoadingCache<String, Instant> loadingCache = CacheBuilder.newBuilder()
.maximumSize(1)
.expireAfterWrite(60, TimeUnit.MINUTES)
.build(new CacheLoader<String, Instant>() {
// The only cached value is associated to "loadtime"
public Instant load(String key) {
final Instant loadTime = getLoadTime();
log.debug("found load time: " + loadTime.toString());
return loadTime;
}
});
private static final Bson fields = getFields();
private static MongoCollection<Document> collection = null;
@Override
@Cacheable("dsm-aggregationhistory-cache")
public List<AggregationInfo> getAggregationHistory(final String dsId) throws DsmException {
log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
final Datasource conf = config.getDatasource();
try {
final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory(dsId, "(collect|transform)"))
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
.map(getMapper())
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
final Instant loadTime = loadingCache.get(LOADTIME);
if (!Objects.equals(Instant.MIN, loadTime)) {
for (final AggregationInfo a : aggregationInfos) {
if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
a.setIndexedVersion(true);
break;
}
}
}
return aggregationInfos;
} catch (Throwable e) {
throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
}
}
private Instant getLoadTime() {
log.warn("querying for metadata load time, not using cache");
return Optional.ofNullable(getCollection().find(queryForLastMetadataLoad()))
.map(d -> d.sort(dbo("system:startHumanDate", -1)).first())
.map(d -> (String) d.getOrDefault("system:startHumanDate", ""))
.map(s -> Instant.parse(s.replaceAll("\\+.*", "Z")))
.orElse(Instant.MIN);
}
private Instant asInstant(final AggregationInfo a) {
return Instant.parse(a.getDate() + "T00:00:00Z");
}
@Override
@CacheEvict(cacheNames = { "dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache" }, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped dsManager aggregation history cache");
}
private Function<Document, AggregationInfo> getMapper() {
return new Function<Document, AggregationInfo>() {
@Override
public AggregationInfo apply(final Document d) {
AggregationInfo info = null;
final AggregationStage stage = AggregationStage.parse(d.getString("system:wfName"));
switch (stage) {
case COLLECT:
CollectionInfo cInfo = new CollectionInfo();
cInfo.setAggregationStage(stage);
cInfo.setCollectionMode(getCollectionMode(d));
cInfo.setNumberOfRecords(getNumberOfRecords(d));
cInfo.setDate(getDate(d));
info = cInfo;
break;
case TRANSFORM:
TransformationInfo tInfo = new TransformationInfo();
tInfo.setAggregationStage(stage);
tInfo.setNumberOfRecords(getNumberOfRecords(d));
tInfo.setDate(getDate(d));
info = tInfo;
break;
}
return info;
}
private CollectionMode getCollectionMode(Document d) {
return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
.map(CollectionMode::valueOf)
.orElseGet(() ->
Optional.ofNullable(d.getString("collectionMode"))
.map(CollectionMode::valueOf)
.orElse(null));
}
private Integer getNumberOfRecords(final Document d) {
final String sinkSize = d.getString("mainlog:sinkSize");
final String total = d.getString("mainlog:total");
if (StringUtils.isNotBlank(sinkSize)) {
return Ints.tryParse(sinkSize);
} else if (StringUtils.isNotBlank(total)) {
return Ints.tryParse(total);
} else {
return -1;
}
}
};
}
private String getDate(final Document d) {
final String dateString = d.getString("system:startHumanDate");
if (StringUtils.isBlank(dateString)) { return ""; }
return DateFormatUtils.format(new DateUtils().parse(dateString), DsmMappingUtils.DATE_FORMAT);
}
private static Bson getFields() {
return fields(
eq("system:wfName", 1),
eq("system:node:SELECT_MODE:selection", 1),
eq("collectionMode", 1),
eq("mainlog:sinkSize", 1),
eq("mainlog:writeOps", 1),
eq("mainlog:total", 1),
eq("system:startHumanDate", 1),
eq("system:profileName", 1));
}
private static BasicDBObject dbo(final String key, final Object value) {
return new BasicDBObject(key, value);
}
private Bson queryForAggregationHistory(final String dsId, final String pattern) {
return and(
eq("parentDatasourceId", dsId),
eq("system:profileFamily", "aggregator"),
eq("system:isCompletedSuccessfully", "true"),
regex("system:wfName", pattern, "i"));
}
private Bson queryForLastMetadataLoad() {
try {
final String contentLoadQuery = config.getContentLoadQuery();
log.debug("parsing content load query: " + contentLoadQuery);
return new ObjectMapper().readValue(contentLoadQuery, BasicDBObject.class);
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
private synchronized MongoCollection<Document> getCollection() {
if (collection == null) {
log.info("inizializing mongodb collection ...");
final Datasource conf = config.getDatasource();
collection = datasourcePublisherMongoClient.getDatabase(conf.getMongoDbName()).getCollection(conf.getMongoCollectionName());
}
return collection;
}
}

9
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java Normal file
View File

@ -0,0 +1,9 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.enabling.datasources.common.DsmException;
public interface ObjectStoreClient {
Long getObjectStoreSize(final String objectStoreId) throws DsmException;
}

37
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java Normal file
View File

@ -0,0 +1,37 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.datasources.common.DsmException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true")
public class ObjectStoreClientImpl implements ObjectStoreClient {
private static final Log log = LogFactory.getLog(ObjectStoreClientImpl.class);
@Autowired
private ObjectStoreService objectStoreService;
@Override
public Long getObjectStoreSize(final String objectStoreId) throws DsmException {
log.debug("get size for objectStore " + objectStoreId);
if (StringUtils.isBlank(objectStoreId)) {
return 0L;
}
try {
final long size = objectStoreService.getSize(objectStoreId);
log.debug("got objectStore size: " + size);
return size;
} catch (ObjectStoreServiceException e) {
throw new DsmException("unable to get size for objectStore " + objectStoreId);
}
}
}

61
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java Normal file
View File

@ -0,0 +1,61 @@
package eu.dnetlib.openaire.dsm.dao;
import java.util.List;
import java.util.Queue;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.DatasourceSearchResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.Header;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
public class ResponseUtils {
public static ApiDetailsResponse apiResponse(final List<ApiDetails> api, final long total) {
final ApiDetailsResponse rsp = new ApiDetailsResponse().setApi(api);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceSnippetResponse snippetResponse(final List<DatasourceSnippetExtended> snippets, final long total) {
final DatasourceSnippetResponse rsp = new DatasourceSnippetResponse(snippets);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceDetailResponse detailsResponse(final List<DatasourceDetails> details, final long total) {
final DatasourceDetailResponse rsp = new DatasourceDetailResponse(details);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceSearchResponse searchResponse(final List<DatasourceInfo> infos, final long total) {
final DatasourceSearchResponse rsp = new DatasourceSearchResponse(infos);
rsp.setHeader(header(total));
return rsp;
}
public static Header header(final Queue<Throwable> errors, final long total) {
return Header.newInsance()
.setExceptions(errors)
.setTotal(total);
}
public static Header header(final long total) {
return header(Lists.newLinkedList(), total);
}
public static SimpleResponse<?> simpleResponse(final List<?> list) {
final SimpleResponse rsp = new SimpleResponse().setResponse(list);;
rsp.setHeader(header(Lists.newLinkedList(), list.size()));
return rsp;
}
}

14
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java Normal file
View File

@ -0,0 +1,14 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.vocabularies.Vocabulary;
public interface VocabularyClient {
Vocabulary getCountries() throws DsmException;
Vocabulary getDatasourceTypologies() throws DsmException;
void dropCache();
}

58
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java Normal file
View File

@ -0,0 +1,58 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.vocabularies.Vocabulary;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
/**
* Created by claudio on 15/09/2017.
*/
@Component
public class VocabularyClientImpl implements VocabularyClient {
private static final Log log = LogFactory.getLog(VocabularyClientImpl.class);
@Autowired
private DnetOpenaireExporterProperties config;
@Override
@Cacheable("vocabularies-cache")
public Vocabulary getCountries() throws DsmException {
return _getVocabulary(config.getVocabularies().getCountriesEndpoint(), Vocabulary.class);
}
@Override
@Cacheable("vocabularies-cache")
public Vocabulary getDatasourceTypologies() throws DsmException {
return _getVocabulary(config.getVocabularies().getDatasourceTypologiesEndpoint(), Vocabulary.class);
}
private <T> T _getVocabulary(final String endpoint, Class<T> clazz) throws DsmException {
final RestTemplate rt = new RestTemplate();
log.info("get vocabulary from " + endpoint);
final ResponseEntity<T> rsp = rt.getForEntity(endpoint, clazz);
if (!rsp.getStatusCode().is2xxSuccessful()) {
throw new DsmException(rsp.getStatusCodeValue(), "unable to read content from " + endpoint);
}
return rsp.getBody();
}
@Override
@CacheEvict(cacheNames = "vocabularies-cache", allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped dsManager vocabulary cache");
}
}

180
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java Normal file
View File

@ -0,0 +1,180 @@
package eu.dnetlib.openaire.dsm.dao.utils;
import java.sql.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapper;
import org.springframework.beans.BeanWrapperImpl;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended;
import eu.dnetlib.openaire.dsm.domain.OrganizationDetails;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.OrganizationDbEntry;
public class DsmMappingUtils {
public static final String DATE_FORMAT = "yyyy-MM-dd";
public static final String ID_SEPARATOR = "::";
public static final String ID_PREFIX = "api_________" + ID_SEPARATOR;
public static String createId(final ApiDetails api) {
return ID_PREFIX + api.getDatasource() + ID_SEPARATOR + RandomStringUtils.randomAlphanumeric(8);
}
public static DatasourceDetails asDetails(final DatasourceDbEntry d) {
final DatasourceDetails details = _convert(d, DatasourceDetails.class);
return details.setOpenaireId(asOpenaireId(details.getId()));
}
public static DatasourceSnippetExtended asSnippetExtended(final DatasourceDbEntry d) {
final DatasourceSnippetExtended ds = new DatasourceSnippetExtended();
ds.setId(d.getId());
ds.setOfficialname(d.getOfficialname());
ds.setEnglishname(d.getEnglishname());
ds.setRegisteredby(d.getRegisteredby());
ds.setWebsiteurl(d.getWebsiteurl());
ds.setEoscDatasourceType(d.getEoscDatasourceType());
ds.setRegistrationdate(d.getRegistrationdate());
ds.setLogoUrl(d.getLogourl());
ds.setDescription(d.getDescription());
ds.setConsentTermsOfUse(d.getConsentTermsOfUse());
ds.setConsentTermsOfUseDate(d.getConsentTermsOfUseDate());
ds.setLastConsentTermsOfUseDate(d.getLastConsentTermsOfUseDate());
ds.setFullTextDownload(d.getFullTextDownload());
if (d.getOrganizations() != null) {
ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet()));
}
ds.setTypology(d.getTypology());
return ds;
}
private static OrganizationDetails asOrganizationDetail(final OrganizationDbEntry o) {
return new OrganizationDetails()
.setCountry(o.getCountry())
.setLegalname(o.getLegalname())
.setLegalshortname(o.getLegalshortname())
.setWebsiteurl(o.getWebsiteurl())
.setLogourl(o.getLogourl());
}
public static ApiDetails asDetails(final ApiDbEntry d) {
return _convert(d, ApiDetails.class);
}
public static ApiDbEntry asDbEntry(final ApiDetails d) {
final ApiDbEntry apiDbEntry = _convert(d, ApiDbEntry.class);
// Need to complete the references among objects, because you know, referential integrity ...
apiDbEntry.getApiParams().forEach(ap -> ap.getId().setApi(apiDbEntry));
return apiDbEntry;
}
public static DatasourceDbEntry asDbEntry(final DatasourceDetails d) {
final DatasourceDbEntry dbe = _convert(d, DatasourceDbEntry.class);
if (dbe.getOrganizations() != null) {
dbe.getOrganizations().forEach(o -> {
final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId();
o.setId(prefix + ID_SEPARATOR + o.getLegalname());
if (o.getDateofcollection() == null) {
o.setDateofcollection(new Date(System.currentTimeMillis()));
}
o.setCollectedfrom(dbe.getCollectedfrom());
});
}
_fix_typology(dbe, d);
return dbe;
}
@Deprecated
private static void _fix_typology(final DatasourceDbEntry dbe, final DatasourceDetails d) {
if (StringUtils.isNotBlank(d.getTypology()) && StringUtils.isBlank(d.getEoscDatasourceType())) {
// THE ORDER IS IMPORTANT: DO NOT CHANGE IT
if (d.getTypology().startsWith("crissystem")) {
dbe.setEoscDatasourceType("CRIS system");
} else if (d.getTypology().startsWith("entityregistry")) {
dbe.setEoscDatasourceType("Registry");
} else if (d.getTypology().startsWith("pubscatalogue") || d.getTypology().equals("websource")) {
dbe.setEoscDatasourceType("Catalogue");
} else if (d.getTypology().contains("journal")) {
dbe.setEoscDatasourceType("Journal archive");
} else if (d.getTypology().startsWith("aggregator")) {
dbe.setEoscDatasourceType("Aggregator");
} else if (d.getTypology().contains("repository")) {
dbe.setEoscDatasourceType("Repository");
} else {
dbe.setEoscDatasourceType("Aggregator");
}
} else if (StringUtils.isBlank(d.getTypology()) && StringUtils.isNotBlank(d.getEoscDatasourceType())) {
if (dbe.getEoscDatasourceType().equals("CRIS system")) {
dbe.setTypology("crissystem");
} else if (dbe.getEoscDatasourceType().equals("Registry")) {
dbe.setTypology("entityregistry");
} else if (dbe.getEoscDatasourceType().equals("Catalogue")) {
dbe.setTypology("pubscatalogue::unknown");
} else if (dbe.getEoscDatasourceType().equals("Journal archive")) {
dbe.setTypology("pubsrepository::journal");
} else if (dbe.getEoscDatasourceType().equals("Aggregator")) {
dbe.setTypology("aggregator");
} else if (dbe.getEoscDatasourceType().equals("Repository")) {
dbe.setTypology("pubsrepository::unknown");
} else {
dbe.setTypology("aggregator");
}
}
}
public static DatasourceDbEntry asDbEntry(final DatasourceDetailsUpdate d) {
return _convert(d, DatasourceDbEntry.class);
}
// HELPERS
private static <T> T _convert(final Object o, final Class<T> clazz) {
final ObjectMapper mapper = new ObjectMapper();
return mapper.convertValue(o, clazz);
}
private static String asOpenaireId(final String id) {
final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR);
final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR);
return prefix + ID_SEPARATOR + AbstractDNetXsltFunctions.md5(md5);
}
public static void copyNonNullProperties(final Object src, final Object target) {
BeanUtils.copyProperties(src, target, getNullPropertyNames(src));
}
public static String[] getNullPropertyNames(final Object source) {
final BeanWrapper src = new BeanWrapperImpl(source);
final java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors();
final Set<String> emptyNames = new HashSet<>();
for (final java.beans.PropertyDescriptor pd : pds) {
final Object srcValue = src.getPropertyValue(pd.getName());
if (srcValue == null) {
emptyNames.add(pd.getName());
}
}
final String[] result = new String[emptyNames.size()];
return emptyNames.toArray(result);
}
}

36
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java Normal file
View File

@ -0,0 +1,36 @@
package eu.dnetlib.openaire.dsm.dao.utils;
/**
* Created by claudio on 20/10/2016.
*/
public class IndexDsInfo {
private final String indexBaseUrl;
private final String indexDsId;
private final String format;
private final String coll;
public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {
this.indexBaseUrl = indexBaseUrl;
this.indexDsId = indexDsId;
this.format = format;
this.coll = coll;
}
public String getIndexBaseUrl() {
return indexBaseUrl;
}
public String getIndexDsId() {
return indexDsId;
}
public String getFormat() {
return format;
}
public String getColl() {
return coll;
}
}

49
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java Normal file
View File

@ -0,0 +1,49 @@
package eu.dnetlib.openaire.dsm.dao.utils;
/**
* Created by claudio on 21/10/2016.
*/
public class IndexRecordsInfo {
private long total;
private long funded;
private String date;
public IndexRecordsInfo() {}
public IndexRecordsInfo(final long total, final long funded, final String date) {
this.total = total;
this.funded = funded;
this.date = date;
}
public long getTotal() {
return total;
}
public IndexRecordsInfo setTotal(final long total) {
this.total = total;
return this;
}
public long getFunded() {
return funded;
}
public IndexRecordsInfo setFunded(final long funded) {
this.funded = funded;
return this;
}
public String getDate() {
return date;
}
public IndexRecordsInfo setDate(final String date) {
this.date = date;
return this;
}
}

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
@ApiModel
@JsonAutoDetect
public class AggregationHistoryResponse extends Response {
@ApiModelProperty(position = 1)
private List<AggregationInfo> aggregationInfo;
public AggregationHistoryResponse(List<AggregationInfo> aggregationInfo) {
super();
this.aggregationInfo = aggregationInfo;
}
public List<AggregationInfo> getAggregationInfo() {
return aggregationInfo;
}
public void setAggregationInfo(List<AggregationInfo> aggregationInfo) {
this.aggregationInfo = aggregationInfo;
}
}

217
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java Normal file
View File

@ -0,0 +1,217 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(value = "Api model", description = "provides information about the datasource API")
public class ApiDetails extends ApiIgnoredProperties {
@ApiModelProperty(position = 0)
private String id = null;
@ApiModelProperty(position = 1)
private String protocol = null;
@ApiModelProperty(position = 2)
private String datasource = null;
@ApiModelProperty(position = 3)
private String contentdescription = null;
@ApiModelProperty(position = 4)
private String eoscDatasourceType = null;
@ApiModelProperty(position = 5)
private String compatibility;
@ApiModelProperty(position = 7)
private String compatibilityOverride;
@ApiModelProperty(position = 8)
private Integer lastCollectionTotal;
@ApiModelProperty(position = 9)
private Date lastCollectionDate;
@ApiModelProperty(position = 10)
private Integer lastAggregationTotal;
@ApiModelProperty(position = 11)
private Date lastAggregationDate;
@ApiModelProperty(position = 12)
private Integer lastDownloadTotal;
@ApiModelProperty(position = 13)
private Date lastDownloadDate;
@ApiModelProperty(position = 14)
private String baseurl;
@ApiModelProperty(position = 15)
protected Boolean removable = false;
@ApiModelProperty(position = 16)
private Set<ApiParamDetails> apiParams;
@ApiModelProperty(position = 17)
private String metadataIdentifierPath = "";
public String getId() {
return id;
}
public String getProtocol() {
return protocol;
}
public String getDatasource() {
return datasource;
}
public String getContentdescription() {
return contentdescription;
}
public String getCompatibility() {
return compatibility;
}
public Integer getLastCollectionTotal() {
return lastCollectionTotal;
}
public Date getLastCollectionDate() {
return lastCollectionDate;
}
public Integer getLastAggregationTotal() {
return lastAggregationTotal;
}
public Date getLastAggregationDate() {
return lastAggregationDate;
}
public Integer getLastDownloadTotal() {
return lastDownloadTotal;
}
public Date getLastDownloadDate() {
return lastDownloadDate;
}
public String getBaseurl() {
return baseurl;
}
public ApiDetails setId(final String id) {
this.id = id;
return this;
}
public ApiDetails setProtocol(final String protocol) {
this.protocol = protocol;
return this;
}
public ApiDetails setDatasource(final String datasource) {
this.datasource = datasource;
return this;
}
public ApiDetails setContentdescription(final String contentdescription) {
this.contentdescription = contentdescription;
return this;
}
public ApiDetails setCompatibility(final String compatibility) {
this.compatibility = compatibility;
return this;
}
public ApiDetails setLastCollectionTotal(final Integer lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
return this;
}
public ApiDetails setLastCollectionDate(final Date lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
return this;
}
public ApiDetails setLastAggregationTotal(final Integer lastAggregationTotal) {
this.lastAggregationTotal = lastAggregationTotal;
return this;
}
public ApiDetails setLastAggregationDate(final Date lastAggregationDate) {
this.lastAggregationDate = lastAggregationDate;
return this;
}
public ApiDetails setLastDownloadTotal(final Integer lastDownloadTotal) {
this.lastDownloadTotal = lastDownloadTotal;
return this;
}
public ApiDetails setLastDownloadDate(final Date lastDownloadDate) {
this.lastDownloadDate = lastDownloadDate;
return this;
}
public ApiDetails setBaseurl(final String baseurl) {
this.baseurl = baseurl;
return this;
}
public Set<ApiParamDetails> getApiParams() {
return apiParams;
}
public void setApiParams(final Set<ApiParamDetails> apiParams) {
this.apiParams = apiParams;
}
public String getCompatibilityOverride() {
return compatibilityOverride;
}
public ApiDetails setCompatibilityOverride(final String compatibilityOverride) {
this.compatibilityOverride = compatibilityOverride;
return this;
}
public Boolean getRemovable() {
return removable;
}
public ApiDetails setRemovable(final Boolean removable) {
this.removable = removable;
return this;
}
public String getMetadataIdentifierPath() {
return metadataIdentifierPath;
}
public ApiDetails setMetadataIdentifierPath(final String metadataIdentifierPath) {
this.metadataIdentifierPath = metadataIdentifierPath;
return this;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public ApiDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
}

24
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java Normal file
View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ApiModel
@JsonAutoDetect
public class ApiDetailsResponse extends Response {
@ApiModelProperty(position = 1)
private List<ApiDetails> api;
public List<ApiDetails> getApi() {
return api;
}
public ApiDetailsResponse setApi(final List<ApiDetails> api) {
this.api = api;
return this;
}
}

72
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java Normal file
View File

@ -0,0 +1,72 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
public abstract class ApiIgnoredProperties {
@JsonIgnore
protected Boolean active = false;
@JsonIgnore
protected String lastCollectionMdid;
@JsonIgnore
protected String lastAggregationMdid;
@JsonIgnore
protected String lastDownloadObjid;
@JsonIgnore
protected String lastValidationJob;
@JsonIgnore
protected boolean compatibilityOverrided;
public Boolean getActive() {
return active;
}
public void setActive(final Boolean active) {
this.active = active;
}
public String getLastCollectionMdid() {
return lastCollectionMdid;
}
public void setLastCollectionMdid(final String lastCollectionMdid) {
this.lastCollectionMdid = lastCollectionMdid;
}
public String getLastAggregationMdid() {
return lastAggregationMdid;
}
public void setLastAggregationMdid(final String lastAggregationMdid) {
this.lastAggregationMdid = lastAggregationMdid;
}
public String getLastDownloadObjid() {
return lastDownloadObjid;
}
public void setLastDownloadObjid(final String lastDownloadObjid) {
this.lastDownloadObjid = lastDownloadObjid;
}
public String getLastValidationJob() {
return lastValidationJob;
}
public void setLastValidationJob(final String lastValidationJob) {
this.lastValidationJob = lastValidationJob;
}
public boolean isCompatibilityOverrided() {
return compatibilityOverrided;
}
public void setCompatibilityOverrided(final boolean compatibilityOverrided) {
this.compatibilityOverrided = compatibilityOverrided;
}
}

24
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java Normal file
View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
public class ApiParamDetails {
protected String param;
protected String value;
public String getParam() {
return param;
}
public void setParam(final String param) {
this.param = param;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
}

25
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import io.swagger.annotations.ApiModel;
/**
* Created by claudio on 29/11/2016.
*/
@ApiModel
@JsonAutoDetect
public class CollectionInfo extends AggregationInfo {
private CollectionMode collectionMode;
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

13
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java Normal file
View File

@ -0,0 +1,13 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
/**
* Created by claudio on 12/09/16.
*/
@ApiModel
@JsonAutoDetect
public enum CollectionMode {
REFRESH, INCREMENTAL
}

28
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java Normal file
View File

@ -0,0 +1,28 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
@ApiModel
@JsonAutoDetect
public class DatasourceDetailResponse extends Response {
@ApiModelProperty(position = 1)
private List<DatasourceDetails> datasourceInfo;
public DatasourceDetailResponse(List<DatasourceDetails> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceDetails> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(List<DatasourceDetails> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

441
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java Normal file
View File

@ -0,0 +1,441 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import javax.persistence.Transient;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
@ApiModel(value = "Datasource model", description = "provides information about the datasource")
public class DatasourceDetails extends DatasourceIgnoredProperties {
@NotBlank
@ApiModelProperty(position = 0)
private String id;
@Transient
@ApiModelProperty(position = 1)
private String openaireId;
@NotBlank
@ApiModelProperty(position = 2)
private String officialname;
@NotBlank
@ApiModelProperty(position = 3)
private String englishname;
@ApiModelProperty(position = 4)
private String websiteurl;
@ApiModelProperty(position = 5)
private String logourl;
@Email
@ApiModelProperty(position = 6)
private String contactemail;
@ApiModelProperty(position = 7)
private Double latitude;
@ApiModelProperty(position = 8)
private Double longitude;
@ApiModelProperty(position = 9)
private String timezone;
@NotBlank
@ApiModelProperty(position = 10)
private String namespaceprefix;
@ApiModelProperty(position = 11)
private String languages;
@ApiModelProperty(position = 12)
private Date dateofvalidation;
@NotBlank
@ApiModelProperty(position = 13)
private String eoscDatasourceType;
@ApiModelProperty(position = 14)
private Date dateofcollection;
@ApiModelProperty(position = 15)
private String platform;
@ApiModelProperty(position = 16)
private String activationId;
@ApiModelProperty(position = 17)
private String description;
@ApiModelProperty(position = 18)
private String issn;
@ApiModelProperty(position = 19)
private String eissn;
@ApiModelProperty(position = 20)
private String lissn;
@Email
@ApiModelProperty(position = 21)
private String registeredby;
@ApiModelProperty(position = 22)
private String subjects;
@ApiModelProperty(position = 23)
protected String aggregator = "OPENAIRE";
@ApiModelProperty(position = 24)
protected String collectedfrom;
@ApiModelProperty(position = 25)
private Boolean managed;
@ApiModelProperty(position = 28)
private Boolean consentTermsOfUse;
@ApiModelProperty(position = 29)
private Boolean fullTextDownload;
@ApiModelProperty(position = 30)
private Date consentTermsOfUseDate;
@ApiModelProperty(position = 31)
private Date lastConsentTermsOfUseDate;
@ApiModelProperty(position = 26)
private Set<OrganizationDetails> organizations;
@ApiModelProperty(position = 27)
private Set<IdentitiesDetails> identities;
@ApiModelProperty(position = 32)
private String status;
@Deprecated
@ApiModelProperty(position = 33)
private String typology;
public String getId() {
return id;
}
public String getOpenaireId() {
return openaireId;
}
public String getOfficialname() {
return officialname;
}
public String getEnglishname() {
return englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getContactemail() {
return contactemail;
}
public Double getLatitude() {
return latitude;
}
public Double getLongitude() {
return longitude;
}
public String getTimezone() {
return timezone;
}
public String getLanguages() {
return languages;
}
public String getNamespaceprefix() {
return namespaceprefix;
}
public Date getDateofvalidation() {
return dateofvalidation;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public Date getDateofcollection() {
return dateofcollection;
}
public String getPlatform() {
return platform;
}
public String getActivationId() {
return activationId;
}
public String getDescription() {
return description;
}
public String getIssn() {
return issn;
}
public String getEissn() {
return eissn;
}
public String getLissn() {
return lissn;
}
public String getRegisteredby() {
return registeredby;
}
public String getSubjects() {
return subjects;
}
public String getAggregator() {
return aggregator;
}
public String getCollectedfrom() {
return collectedfrom;
}
public Boolean getManaged() {
return managed;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public Set<OrganizationDetails> getOrganizations() {
return organizations;
}
public Set<IdentitiesDetails> getIdentities() {
return identities;
}
public DatasourceDetails setId(final String id) {
this.id = id;
return this;
}
public DatasourceDetails setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
return this;
}
public DatasourceDetails setOfficialname(final String officialname) {
this.officialname = officialname;
return this;
}
public DatasourceDetails setEnglishname(final String englishname) {
this.englishname = englishname;
return this;
}
public DatasourceDetails setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public DatasourceDetails setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public DatasourceDetails setContactemail(final String contactemail) {
this.contactemail = contactemail;
return this;
}
public DatasourceDetails setLatitude(final Double latitude) {
this.latitude = latitude;
return this;
}
public DatasourceDetails setLongitude(final Double longitude) {
this.longitude = longitude;
return this;
}
public DatasourceDetails setTimezone(final String timezone) {
this.timezone = timezone;
return this;
}
public DatasourceDetails setLanguages(final String languages) {
this.languages = languages;
return this;
}
public DatasourceDetails setNamespaceprefix(final String namespaceprefix) {
this.namespaceprefix = namespaceprefix;
return this;
}
public DatasourceDetails setDateofvalidation(final Date dateofvalidation) {
this.dateofvalidation = dateofvalidation;
return this;
}
public DatasourceDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
public DatasourceDetails setDateofcollection(final Date dateofcollection) {
this.dateofcollection = dateofcollection;
return this;
}
public DatasourceDetails setPlatform(final String platform) {
this.platform = platform;
return this;
}
public DatasourceDetails setActivationId(final String activationId) {
this.activationId = activationId;
return this;
}
public DatasourceDetails setDescription(final String description) {
this.description = description;
return this;
}
public DatasourceDetails setIssn(final String issn) {
this.issn = issn;
return this;
}
public DatasourceDetails setEissn(final String eissn) {
this.eissn = eissn;
return this;
}
public DatasourceDetails setLissn(final String lissn) {
this.lissn = lissn;
return this;
}
public DatasourceDetails setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
return this;
}
public DatasourceDetails setSubjects(final String subjects) {
this.subjects = subjects;
return this;
}
public DatasourceDetails setAggregator(final String aggregator) {
this.aggregator = aggregator;
return this;
}
public DatasourceDetails setCollectedfrom(final String collectedfrom) {
this.collectedfrom = collectedfrom;
return this;
}
public DatasourceDetails setManaged(final Boolean managed) {
this.managed = managed;
return this;
}
public DatasourceDetails setOrganizations(final Set<OrganizationDetails> organizations) {
this.organizations = organizations;
return this;
}
public DatasourceDetails setIdentities(final Set<IdentitiesDetails> identities) {
this.identities = identities;
return this;
}
public DatasourceDetails setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public DatasourceDetails setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public String getStatus() {
return status;
}
public DatasourceDetails setStatus(final String status) {
this.status = status;
return this;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public DatasourceDetails setTypology(final String typology) {
this.typology = typology;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
}

255
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java Normal file
View File

@ -0,0 +1,255 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
@ApiModel(value = "Datasource updatable fields model", description = "provides information about the datasource field that can be updated")
public class DatasourceDetailsUpdate {
@NotBlank
@ApiModelProperty(position = 0)
private String id;
@NotBlank
@ApiModelProperty(position = 2)
private String officialname;
@NotBlank
@ApiModelProperty(position = 3)
private String englishname;
@ApiModelProperty(position = 4)
private String websiteurl;
@ApiModelProperty(position = 5)
private String logourl;
@Email
@ApiModelProperty(position = 6)
private String contactemail;
@ApiModelProperty(position = 7)
private Double latitude;
@ApiModelProperty(position = 8)
private Double longitude;
@ApiModelProperty(position = 9)
private String timezone;
@NotBlank
@ApiModelProperty(position = 13)
private String eoscDatasourceType;
@ApiModelProperty(position = 15)
private String platform;
@ApiModelProperty(position = 17)
private String description;
@Email
@ApiModelProperty(position = 21)
private String registeredby;
@ApiModelProperty(position = 25)
private Boolean managed;
@ApiModelProperty(position = 27)
private Set<IdentitiesDetails> identities;
@ApiModelProperty(position = 28)
private Boolean consentTermsOfUse;
@ApiModelProperty(position = 29)
private Date consentTermsOfUseDate;
@ApiModelProperty(position = 29)
private Date lastConsentTermsOfUseDate;
@ApiModelProperty(position = 31)
private Boolean fullTextDownload;
public String getId() {
return id;
}
public String getOfficialname() {
return officialname;
}
public String getEnglishname() {
return englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getContactemail() {
return contactemail;
}
public Double getLatitude() {
return latitude;
}
public Double getLongitude() {
return longitude;
}
public String getTimezone() {
return timezone;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public String getPlatform() {
return platform;
}
public String getDescription() {
return description;
}
public String getRegisteredby() {
return registeredby;
}
public Boolean getManaged() {
return managed;
}
public Set<IdentitiesDetails> getIdentities() {
return identities;
}
public DatasourceDetailsUpdate setId(final String id) {
this.id = id;
return this;
}
public DatasourceDetailsUpdate setOfficialname(final String officialname) {
this.officialname = officialname;
return this;
}
public DatasourceDetailsUpdate setEnglishname(final String englishname) {
this.englishname = englishname;
return this;
}
public DatasourceDetailsUpdate setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public DatasourceDetailsUpdate setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public DatasourceDetailsUpdate setContactemail(final String contactemail) {
this.contactemail = contactemail;
return this;
}
public DatasourceDetailsUpdate setLatitude(final Double latitude) {
this.latitude = latitude;
return this;
}
public DatasourceDetailsUpdate setLongitude(final Double longitude) {
this.longitude = longitude;
return this;
}
public DatasourceDetailsUpdate setTimezone(final String timezone) {
this.timezone = timezone;
return this;
}
public DatasourceDetailsUpdate setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
public DatasourceDetailsUpdate setPlatform(final String platform) {
this.platform = platform;
return this;
}
public DatasourceDetailsUpdate setDescription(final String description) {
this.description = description;
return this;
}
public DatasourceDetailsUpdate setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
return this;
}
public DatasourceDetailsUpdate setManaged(final Boolean managed) {
this.managed = managed;
return this;
}
public DatasourceDetailsUpdate setIdentities(final Set<IdentitiesDetails> identities) {
this.identities = identities;
return this;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public DatasourceDetailsUpdate setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public DatasourceDetailsUpdate setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
}

196
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java Normal file
View File

@ -0,0 +1,196 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import com.fasterxml.jackson.annotation.JsonIgnore;
public abstract class DatasourceIgnoredProperties {
@JsonIgnore
protected String od_contenttypes;
@JsonIgnore
protected String provenanceaction;
@JsonIgnore
protected Date releasestartdate;
@JsonIgnore
protected Date releaseenddate;
@JsonIgnore
protected String missionstatementurl;
@JsonIgnore
protected Boolean dataprovider;
@JsonIgnore
protected Boolean serviceprovider;
@JsonIgnore
protected String databaseaccesstype;
@JsonIgnore
protected String datauploadtype;
@JsonIgnore
protected String databaseaccessrestriction;
@JsonIgnore
protected String datauploadrestriction;
@JsonIgnore
protected Boolean versioning;
@JsonIgnore
protected String citationguidelineurl;
@JsonIgnore
protected String qualitymanagementkind;
@JsonIgnore
protected String pidsystems;
@JsonIgnore
protected String certificates;
@JsonIgnore
protected Date registrationdate;
public String getOd_contenttypes() {
return od_contenttypes;
}
public void setOd_contenttypes(final String od_contenttypes) {
this.od_contenttypes = od_contenttypes;
}
public String getProvenanceaction() {
return provenanceaction;
}
public void setProvenanceaction(final String provenanceaction) {
this.provenanceaction = provenanceaction;
}
public Date getReleasestartdate() {
return releasestartdate;
}
public void setReleasestartdate(final Date releasestartdate) {
this.releasestartdate = releasestartdate;
}
public Date getReleaseenddate() {
return releaseenddate;
}
public void setReleaseenddate(final Date releaseenddate) {
this.releaseenddate = releaseenddate;
}
public String getMissionstatementurl() {
return missionstatementurl;
}
public void setMissionstatementurl(final String missionstatementurl) {
this.missionstatementurl = missionstatementurl;
}
public Boolean getDataprovider() {
return dataprovider;
}
public void setDataprovider(final Boolean dataprovider) {
this.dataprovider = dataprovider;
}
public Boolean getServiceprovider() {
return serviceprovider;
}
public void setServiceprovider(final Boolean serviceprovider) {
this.serviceprovider = serviceprovider;
}
public String getDatabaseaccesstype() {
return databaseaccesstype;
}
public void setDatabaseaccesstype(final String databaseaccesstype) {
this.databaseaccesstype = databaseaccesstype;
}
public String getDatauploadtype() {
return datauploadtype;
}
public void setDatauploadtype(final String datauploadtype) {
this.datauploadtype = datauploadtype;
}
public String getDatabaseaccessrestriction() {
return databaseaccessrestriction;
}
public void setDatabaseaccessrestriction(final String databaseaccessrestriction) {
this.databaseaccessrestriction = databaseaccessrestriction;
}
public String getDatauploadrestriction() {
return datauploadrestriction;
}
public void setDatauploadrestriction(final String datauploadrestriction) {
this.datauploadrestriction = datauploadrestriction;
}
public Boolean getVersioning() {
return versioning;
}
public void setVersioning(final Boolean versioning) {
this.versioning = versioning;
}
public String getCitationguidelineurl() {
return citationguidelineurl;
}
public void setCitationguidelineurl(final String citationguidelineurl) {
this.citationguidelineurl = citationguidelineurl;
}
public String getQualitymanagementkind() {
return qualitymanagementkind;
}
public void setQualitymanagementkind(final String qualitymanagementkind) {
this.qualitymanagementkind = qualitymanagementkind;
}
public String getPidsystems() {
return pidsystems;
}
public void setPidsystems(final String pidsystems) {
this.pidsystems = pidsystems;
}
public String getCertificates() {
return certificates;
}
public void setCertificates(final String certificates) {
this.certificates = certificates;
}
public Date getRegistrationdate() {
return registrationdate;
}
public void setRegistrationdate(final Date registrationdate) {
this.registrationdate = registrationdate;
}
}

124
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java Normal file
View File

@ -0,0 +1,124 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(value = "Datasource info model", description = "provides information about the datasource and its aggregation status")
public class DatasourceInfo {
@ApiModelProperty(position = 0)
private long indexRecords;
@ApiModelProperty(position = 1)
private long fundedContent;
@ApiModelProperty(position = 2)
private long fulltexts;
@ApiModelProperty(position = 3)
private String lastIndexingDate;
@ApiModelProperty(position = 4)
private String firstHarvestDate;
@ApiModelProperty(position = 5)
private DatasourceDetails datasource;
@ApiModelProperty(position = 6)
private AggregationInfo lastCollection;
@ApiModelProperty(position = 7)
private AggregationInfo lastTransformation;
@ApiModelProperty(position = 8)
private List<AggregationInfo> aggregationHistory;
public DatasourceInfo() {
super();
}
public DatasourceInfo setIndexRecords(final long indexRecords) {
this.indexRecords = indexRecords;
return this;
}
public DatasourceInfo setFundedContent(final long fundedContent) {
this.fundedContent = fundedContent;
return this;
}
public DatasourceInfo setFulltexts(final long fulltexts) {
this.fulltexts = fulltexts;
return this;
}
public DatasourceInfo setLastIndexingDate(final String lastIndexingDate) {
this.lastIndexingDate = lastIndexingDate;
return this;
}
public DatasourceInfo setAggregationHistory(final List<AggregationInfo> aggregationHistory) {
this.aggregationHistory = aggregationHistory;
return this;
}
public DatasourceInfo setLastCollection(final AggregationInfo lastCollection) {
this.lastCollection = lastCollection;
return this;
}
public DatasourceInfo setLastTransformation(final AggregationInfo lastTransformation) {
this.lastTransformation = lastTransformation;
return this;
}
public long getIndexRecords() {
return indexRecords;
}
public long getFundedContent() {
return fundedContent;
}
public long getFulltexts() {
return fulltexts;
}
public String getLastIndexingDate() {
return lastIndexingDate;
}
public List<AggregationInfo> getAggregationHistory() {
return aggregationHistory;
}
public AggregationInfo getLastCollection() {
return lastCollection;
}
public AggregationInfo getLastTransformation() {
return lastTransformation;
}
public DatasourceDetails getDatasource() {
return datasource;
}
public DatasourceInfo setDatasource(final DatasourceDetails datasource) {
this.datasource = datasource;
return this;
}
public String getFirstHarvestDate() {
return firstHarvestDate;
}
public void setFirstHarvestDate(final String firstHarvestDate) {
this.firstHarvestDate = firstHarvestDate;
}
}

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.common.collect.Lists;
import io.swagger.annotations.ApiModel;
@ApiModel
@JsonAutoDetect
public class DatasourceResponse<T> extends Response {
private List<T> datasourceInfo = Lists.newArrayList();
public DatasourceResponse<T> addDatasourceInfo(final T datasourceInfo) {
getDatasourceInfo().add(datasourceInfo);
return this;
}
public List<T> getDatasourceInfo() {
return datasourceInfo;
}
public DatasourceResponse<T> setDatasourceInfo(final List<T> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
return this;
}
}

28
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java Normal file
View File

@ -0,0 +1,28 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
@ApiModel
@JsonAutoDetect
public class DatasourceSearchResponse extends Response {
@ApiModelProperty(position = 1)
private List<DatasourceInfo> datasourceInfo;
public DatasourceSearchResponse(List<DatasourceInfo> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceInfo> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(List<DatasourceInfo> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

49
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java Normal file
View File

@ -0,0 +1,49 @@
package eu.dnetlib.openaire.dsm.domain;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(value = "Datasource model", description = "provides information about the datasource")
public class DatasourceSnippet {
@NotBlank
@ApiModelProperty(position = 0)
private String id;
@NotBlank
@ApiModelProperty(position = 2)
private String officialname;
@NotBlank
@ApiModelProperty(position = 3)
private String englishname;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public String getEnglishname() {
return englishname;
}
public void setEnglishname(final String englishname) {
this.englishname = englishname;
}
}

198
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java Normal file
View File

@ -0,0 +1,198 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.Date;
import java.util.Set;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(value = "Datasource model", description = "provides extended information about the datasource")
public class DatasourceSnippetExtended {
@NotBlank
@ApiModelProperty(position = 0)
private String id;
@NotBlank
@ApiModelProperty(position = 2)
private String officialname;
@NotBlank
@ApiModelProperty(position = 3)
private String englishname;
@ApiModelProperty(position = 4)
private String websiteurl;
@Email
@ApiModelProperty(position = 5)
private String registeredby;
@ApiModelProperty(position = 6)
private Date registrationdate;
@ApiModelProperty(position = 7)
private String eoscDatasourceType;
@ApiModelProperty(position = 8)
private String logoUrl;
@ApiModelProperty(position = 9)
private String description;
@ApiModelProperty(position = 10)
private Boolean consentTermsOfUse;
@ApiModelProperty(position = 11)
private Date consentTermsOfUseDate;
@ApiModelProperty(position = 12)
private Date lastConsentTermsOfUseDate;
@ApiModelProperty(position = 13)
private Boolean fullTextDownload;
@ApiModelProperty(position = 14)
private Set<OrganizationDetails> organizations;
@Deprecated
@ApiModelProperty(position = 15)
private String typology;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public String getEnglishname() {
return englishname;
}
public void setEnglishname(final String englishname) {
this.englishname = englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public void setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
}
public String getRegisteredby() {
return registeredby;
}
public void setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
}
public Date getRegistrationdate() {
return registrationdate;
}
public void setRegistrationdate(final Date registrationdate) {
this.registrationdate = registrationdate;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getLogoUrl() {
return logoUrl;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public DatasourceSnippetExtended setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public DatasourceSnippetExtended setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public DatasourceSnippetExtended setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
return this;
}
public String getDescription() {
return description;
}
public DatasourceSnippetExtended setDescription(final String description) {
this.description = description;
return this;
}
public Set<OrganizationDetails> getOrganizations() {
return organizations;
}
public DatasourceSnippetExtended setOrganizations(final Set<OrganizationDetails> organizations) {
this.organizations = organizations;
return this;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public DatasourceSnippetExtended setTypology(final String typology) {
this.typology = typology;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
}

28
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java Normal file
View File

@ -0,0 +1,28 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
@ApiModel
@JsonAutoDetect
public class DatasourceSnippetResponse extends Response {
@ApiModelProperty(position = 1)
private List<DatasourceSnippetExtended> datasourceInfo;
public DatasourceSnippetResponse(List<DatasourceSnippetExtended> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceSnippetExtended> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(List<DatasourceSnippetExtended> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

44
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java Normal file
View File

@ -0,0 +1,44 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
@JsonAutoDetect
@ApiModel(value = "Filter name", description = "List of the field names used to filter datasources")
public enum FilterName {
id,
managed,
collectedfrom, // exact match
officialname,
englishname,
websiteurl,
contactemail,
registeredby,
eoscDatasourceType,
platform, // like match
country; // exact match on related organization
public static FilterType type(final FilterName filterName) {
switch (filterName) {
case id:
case managed:
case collectedfrom:
return FilterType.exact;
case officialname:
case englishname:
case websiteurl:
case contactemail:
case registeredby:
case eoscDatasourceType:
case platform:
return FilterType.search;
case country:
return FilterType.searchOrgs;
default:
throw new IllegalStateException("unmapped filter type for: " + filterName);
}
}
}

5
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java Normal file
View File

@ -0,0 +1,5 @@
package eu.dnetlib.openaire.dsm.domain;
public enum FilterType {
exact, search, searchOrgs
}

115
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java Normal file
View File

@ -0,0 +1,115 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import java.util.Queue;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.Lists;
import com.google.gson.GsonBuilder;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ApiModel
@JsonAutoDetect
public class Header {
@ApiModelProperty(position = 0)
private long total;
@ApiModelProperty(position = 1)
private int page;
@ApiModelProperty(position = 2)
private int size;
@ApiModelProperty(position = 3)
private long time;
@ApiModelProperty(position = 4)
private int statusCode;
@ApiModelProperty(position = 5)
private List<String> errors = Lists.newArrayList();
@JsonIgnore
private Queue<Throwable> exceptions = Lists.newLinkedList();
public static Header newInsance() {
return new Header();
}
public Header() {
}
public long getTime() {
return time;
}
public Header setTime(final long time) {
this.time = time;
return this;
}
public int getStatusCode() {
return statusCode;
}
public Header setStatusCode(final int statusCode) {
this.statusCode = statusCode;
return this;
}
public long getTotal() {
return total;
}
public int getPage() {
return page;
}
public int getSize() {
return size;
}
public Header setPage(final int page) {
this.page = page;
return this;
}
public Header setSize(final int size) {
this.size = size;
return this;
}
public Header setTotal(final long total) {
this.total = total;
return this;
}
public Queue<Throwable> getExceptions() {
return exceptions;
}
public Header setExceptions(final Queue<Throwable> exceptions) {
this.exceptions = exceptions;
return this;
}
public List<String> getErrors() {
return getExceptions().stream()
.map(Throwable::getMessage)
.collect(Collectors.toList());
}
public Header setErrors(final List<String> errors) {
this.errors = errors;
return this;
}
public String toJson() {
return new GsonBuilder().setPrettyPrinting().create().toJson(this);
}
}

32
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java Normal file
View File

@ -0,0 +1,32 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
@ApiModel
@JsonAutoDetect
public class IdentitiesDetails {
private String pid;
private String issuertype;
public String getPid() {
return pid;
}
public String getIssuertype() {
return issuertype;
}
public IdentitiesDetails setPid(final String pid) {
this.pid = pid;
return this;
}
public IdentitiesDetails setIssuertype(final String issuertype) {
this.issuertype = issuertype;
return this;
}
}

76
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java Normal file
View File

@ -0,0 +1,76 @@
package eu.dnetlib.openaire.dsm.domain;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(value = "Organization info model", description = "provides information about the organization")
public class OrganizationDetails extends OrganizationIgnoredProperties {
@ApiModelProperty(position = 0)
private String legalshortname;
@NotBlank
@ApiModelProperty(position = 1)
private String legalname;
@ApiModelProperty(position = 2)
private String websiteurl;
@ApiModelProperty(position = 3)
private String logourl;
@NotBlank
@ApiModelProperty(position = 4)
private String country;
public String getLegalshortname() {
return legalshortname;
}
public String getLegalname() {
return legalname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getCountry() {
return country;
}
public OrganizationDetails setLegalshortname(final String legalshortname) {
this.legalshortname = legalshortname;
return this;
}
public OrganizationDetails setLegalname(final String legalname) {
this.legalname = legalname;
return this;
}
public OrganizationDetails setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public OrganizationDetails setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public OrganizationDetails setCountry(final String country) {
this.country = country;
return this;
}
}

64
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java Normal file
View File

@ -0,0 +1,64 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.Date;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore;
public class OrganizationIgnoredProperties {
@JsonIgnore
protected String id;
@JsonIgnore
protected String collectedfrom;
@JsonIgnore
protected Date dateofcollection;
@JsonIgnore
protected String provenanceaction;
@JsonIgnore
protected Set<?> datasources;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getCollectedfrom() {
return collectedfrom;
}
public void setCollectedfrom(final String collectedfrom) {
this.collectedfrom = collectedfrom;
}
public Date getDateofcollection() {
return dateofcollection;
}
public void setDateofcollection(final Date dateofcollection) {
this.dateofcollection = dateofcollection;
}
public String getProvenanceaction() {
return provenanceaction;
}
public void setProvenanceaction(final String provenanceaction) {
this.provenanceaction = provenanceaction;
}
public Set<?> getDatasources() {
return datasources;
}
public void setDatasources(final Set<?> datasources) {
this.datasources = datasources;
}
}

96
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java Normal file
View File

@ -0,0 +1,96 @@
package eu.dnetlib.openaire.dsm.domain;
public class RegisteredDatasourceInfo {
private String id;
private String officialName;
private String englishName;
private String organization;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
private String lastCollectionDate;
private long lastCollectionTotal;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialName() {
return officialName;
}
public void setOfficialName(final String officialName) {
this.officialName = officialName;
}
public String getEnglishName() {
return englishName;
}
public void setEnglishName(final String englishName) {
this.englishName = englishName;
}
public String getOrganization() {
return organization;
}
public void setOrganization(final String organization) {
this.organization = organization;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
return registeredBy;
}
public void setRegisteredBy(final String registeredBy) {
this.registeredBy = registeredBy;
}
public String getRegistrationDate() {
return registrationDate;
}
public void setRegistrationDate(final String registrationDate) {
this.registrationDate = registrationDate;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(final String compatibility) {
this.compatibility = compatibility;
}
public String getLastCollectionDate() {
return lastCollectionDate;
}
public void setLastCollectionDate(final String lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
}
public long getLastCollectionTotal() {
return lastCollectionTotal;
}
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

20
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java Normal file
View File

@ -0,0 +1,20 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.HashMap;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
@JsonAutoDetect
@ApiModel(value = "Request filter", description = "field name and value pairs")
public class RequestFilter extends HashMap<FilterName, Object> {
/**
*
*/
private static final long serialVersionUID = 5501969842482508379L;
public RequestFilter() {}
}

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java Normal file
View File

@ -0,0 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public enum RequestSort {
id, officialname, dateofvalidation, registrationdate
}

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java Normal file
View File

@ -0,0 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public enum RequestSortOrder {
ASCENDING, DESCENDING
}

29
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java Normal file
View File

@ -0,0 +1,29 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@JsonAutoDetect
@ApiModel(
value = "Api response model",
description = "Api response model, provides a response header")
public class Response {
@ApiModelProperty(position = 0)
private Header header;
public Response() {
this.header = new Header();
}
public Header getHeader() {
return header;
}
public Response setHeader(final Header header) {
this.header = header;
return this;
}
}

25
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@ApiModel
@JsonAutoDetect
public class SimpleResponse<T> extends Response {
@ApiModelProperty(position = 1)
private List<T> response;
public List<T> getResponse() {
return response;
}
public SimpleResponse<T> setResponse(final List<T> response) {
this.response = response;
return this;
}
}

15
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java Normal file
View File

@ -0,0 +1,15 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import io.swagger.annotations.ApiModel;
/**
* Created by claudio on 29/11/2016.
*/
@ApiModel
@JsonAutoDetect
public class TransformationInfo extends AggregationInfo {
}

Some files were not shown because too many files have changed in this diff Show More