diff --git a/apps/dnet-exporter-api/README.md b/apps/dnet-exporter-api/README.md new file mode 100644 index 00000000..a0a8d3f5 --- /dev/null +++ b/apps/dnet-exporter-api/README.md @@ -0,0 +1,6 @@ +SpringBoot application implementing OpenAIRE REST API to manage +- Datasources +- Contexts +- Communities +- Funders +- Projects \ No newline at end of file diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml new file mode 100644 index 00000000..122aed3d --- /dev/null +++ b/apps/dnet-exporter-api/pom.xml @@ -0,0 +1,149 @@ + + + + eu.dnetlib.dhp + apps + 3.2.6-SNAPSHOT + ../ + + + 4.0.0 + dnet-exporter-api + jar + + + + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-cache + + + eu.dnetlib + cnr-rmi-api + [2.0.0,3.0.0) + + + org.apache.cxf + cxf-rt-transports-http + 3.1.5 + + + eu.dnetlib + cnr-service-common + [2.0.0,3.0.0) + + + eu.dnetlib + dnet-openaireplus-mapping-utils + [6.3.0,7.0.0) + + + com.sun.jersey + jersey-client + + + eu.dnetlib + dnet-hadoop-commons + + + + + eu.dnetlib + dnet-objectstore-rmi + [2.0.0,3.0.0) + + + org.apache.solr + solr-solrj + 7.5.0 + + + + org.postgresql + postgresql + + + + org.apache.commons + commons-dbcp2 + + + + org.antlr + stringtemplate + 3.2.1 + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpmime + + + net.sf.supercsv + super-csv + 2.4.0 + + + com.google.code.gson + gson + + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + + + joda-time + joda-time + 2.8.2 + + + + org.mongodb + mongo-java-driver + 3.4.2 + + + + eu.dnetlib + dnet-datasource-manager-common + [2.0.1,3.0.0) + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.apache.maven.plugins + maven-help-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java new file mode 100644 index 00000000..cac25806 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/CacheCustomizer.java @@ -0,0 +1,29 @@ +package eu.dnetlib; + +import org.springframework.boot.autoconfigure.cache.CacheManagerCustomizer; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.stereotype.Component; + +import static java.util.Arrays.asList; + +@Component +public class CacheCustomizer implements CacheManagerCustomizer { + + @Override + public void customize(final ConcurrentMapCacheManager cacheManager) { + cacheManager.setCacheNames( + asList( + "fundingpath-ids", + "indexdsinfo-cache", + "objectstoreid-cache", + "context-cache", + "context-cache-funder", + "context-cache-community", + "dsm-aggregationhistory-cache", + "dsm-firstharvestdate-cache", + "vocabularies-cache", + "community-cache", + "info")); + } + +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java new file mode 100644 index 00000000..8af6272c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterApplication.java @@ -0,0 +1,110 @@ +package eu.dnetlib; + +import static springfox.documentation.builders.RequestHandlerSelectors.basePackage; + +import java.time.LocalDate; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; + +import eu.dnetlib.DnetOpenaireExporterProperties.Swagger; +import eu.dnetlib.common.app.AbstractDnetApp; +import eu.dnetlib.openaire.community.CommunityApiController; +import eu.dnetlib.openaire.context.ContextApiController; +import eu.dnetlib.openaire.dsm.DsmApiController; +import eu.dnetlib.openaire.funders.FundersApiController; +import eu.dnetlib.openaire.info.InfoController; +import eu.dnetlib.openaire.project.ProjectsController; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +@EnableCaching +@EnableScheduling +@EnableSwagger2 +@SpringBootApplication +@EnableAutoConfiguration(exclude = { + SolrAutoConfiguration.class +}) +public class DNetOpenaireExporterApplication extends AbstractDnetApp { + + public static final String V1 = "1.0.0"; + + public static void main(final String[] args) throws Exception { + SpringApplication.run(DNetOpenaireExporterApplication.class, args); + } + + @Autowired + private DnetOpenaireExporterProperties config; + + @Bean + public Docket dsm() { + return _docket("Datasource Manager", DsmApiController.class.getPackage().getName(), config.getSwaggerDsm(), V1); + } + + @Bean + public Docket projects() { + return _docket("OpenAIRE Projects", ProjectsController.class.getPackage().getName(), config.getSwaggerProjects(), V1); + } + + @Bean + public Docket funders() { + return _docket("OpenAIRE Funders", FundersApiController.class.getPackage().getName(), config.getSwaggerFunders(), V1); + } + + @Bean + public Docket communities() { + return _docket("OpenAIRE Communities", CommunityApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1); + } + + @Bean + public Docket contexts() { + return _docket("OpenAIRE Contexts", ContextApiController.class.getPackage().getName(), config.getSwaggerCommunities(), V1); + } + + private Docket _docket(final String groupName, final String controllerPackage, final Swagger swag, final String version) { + final Docket d = new Docket(DocumentationType.SWAGGER_2); + configSwagger(d, groupName, controllerPackage, swag, version); + return d; + } + + @Override + protected void configSwagger(final Docket docket) { + configSwagger(docket, "OpenAIRE Info", InfoController.class.getPackage().getName(), config.getSwaggerInfo(), V1); + } + + private void configSwagger(final Docket docket, final String groupName, final String controllerPackage, final Swagger swag, final String version) { + docket + .groupName(groupName) + .select() + .apis(basePackage(controllerPackage)) + .build() + .directModelSubstitute(LocalDate.class, java.sql.Date.class) + .apiInfo(apiInfo(swag, version)); + } + + private ApiInfo apiInfo(final Swagger swag, final String version) { + return new ApiInfoBuilder() + .title(swag.getApiTitle()) + .description(swag.getApiDescription()) + .license(swag.getApiLicense()) + .licenseUrl(swag.getApiLicenseUrl()) + .termsOfServiceUrl("") + .version(version) + .contact(new Contact( + swag.getApiContactName(), + swag.getApiContactUrl(), + swag.getApiContactEmail())) + .build(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java new file mode 100644 index 00000000..24674d3a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DNetOpenaireExporterConfiguration.java @@ -0,0 +1,106 @@ +package eu.dnetlib; + +import javax.sql.DataSource; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.cxf.endpoint.Client; +import org.apache.cxf.frontend.ClientProxy; +import org.apache.cxf.jaxws.JaxWsProxyFactoryBean; +import org.apache.cxf.transport.http.HTTPConduit; +import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.ServerAddress; + +import eu.dnetlib.DnetOpenaireExporterProperties.Jdbc; +import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; + +/** + * Created by claudio on 07/07/2017. + */ +@Configuration +public class DNetOpenaireExporterConfiguration { + + private static final Log log = LogFactory.getLog(DNetOpenaireExporterConfiguration.class); + + @Autowired + private DnetOpenaireExporterProperties props; + + @Bean + public ISLookUpService getLookUpService() { + return getServiceStub(ISLookUpService.class, props.getIsLookupUrl()); + } + + @Bean + public ObjectStoreService getObjectStoreService() { + return getServiceStub(ObjectStoreService.class, props.getObjectStoreServiceUrl()); + } + + @Bean + public ISRegistryService getRegistryService() { + return getServiceStub(ISRegistryService.class, props.getIsRegistryServiceUrl()); + } + + @SuppressWarnings("unchecked") + private T getServiceStub(final Class clazz, final String endpoint) { + log.info(String.format("Initializing service stub %s, endpoint %s", clazz.toString(), endpoint)); + final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean(); + jaxWsProxyFactory.setServiceClass(clazz); + jaxWsProxyFactory.setAddress(endpoint); + + final T service = (T) jaxWsProxyFactory.create(); + + final Client client = ClientProxy.getClient(service); + if (client != null) { + final HTTPConduit conduit = (HTTPConduit) client.getConduit(); + final HTTPClientPolicy policy = new HTTPClientPolicy(); + + log.info(String.format("setting connectTimeout to %s, receiveTimeout to %s for service %s", props.getCxfClientConnectTimeout(), props + .getCxfClientReceiveTimeout(), clazz.getCanonicalName())); + + policy.setConnectionTimeout(props.getCxfClientConnectTimeout()); + policy.setReceiveTimeout(props.getCxfClientReceiveTimeout()); + conduit.setClient(policy); + } + + return service; + } + + @Bean + public DataSource getSqlDataSource() { + final Jdbc jdbc = props.getJdbc(); + return getDatasource(jdbc.getDriverClassName(), jdbc.getUrl(), jdbc.getUser(), jdbc.getPwd(), jdbc.getMinIdle(), jdbc.getMaxRows()); + } + + private BasicDataSource getDatasource(final String driverClassName, + final String jdbcUrl, + final String jdbcUser, + final String jdbcPwd, + final int jdbcMinIdle, + final int jdbcMaxIdle) { + final BasicDataSource d = new BasicDataSource(); + d.setDriverClassName(driverClassName); + d.setUrl(jdbcUrl); + d.setUsername(jdbcUser); + d.setPassword(jdbcPwd); + d.setMinIdle(jdbcMinIdle); + d.setMaxIdle(jdbcMaxIdle); + return d; + } + + @Bean + public MongoClient getMongoClient() { + return new MongoClient( + new ServerAddress(props.getDatasource().getMongoHost(), props.getDatasource().getMongoPort()), + MongoClientOptions.builder().connectionsPerHost(props.getDatasource().getMongoConnectionsPerHost()).build()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java new file mode 100644 index 00000000..fbdaf701 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/DnetOpenaireExporterProperties.java @@ -0,0 +1,548 @@ +package eu.dnetlib; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +/** + * Created by Alessia Bardi on 31/03/17. + * + * @author Alessia Bardi, Claudio Atzori + */ +@Configuration +@PropertySource("classpath:global.properties") +@ConfigurationProperties(prefix = "openaire.exporter") +public class DnetOpenaireExporterProperties { + + // ISLOOKUP + private ClassPathResource findSolrIndexUrl; + private ClassPathResource findIndexDsInfo; + private ClassPathResource findObjectStore; + private ClassPathResource findFunderContexts; + private ClassPathResource findCommunityContexts; + private ClassPathResource findContextProfiles; + private ClassPathResource findContextProfilesByType; + private ClassPathResource getRepoProfile; + + @Value("${openaire.exporter.contentLoadQuery}") + private String contentLoadQuery; + + private String isLookupUrl; + private String objectStoreServiceUrl; + private String isRegistryServiceUrl; + + private int requestWorkers = 100; + private int requestTimeout = 10; + + private int cxfClientConnectTimeout = 120; + private int cxfClientReceiveTimeout = 120; + + private Datasource datasource; + private Project project; + private Jdbc jdbc; + private Swagger swaggerDsm; + private Swagger swaggerProjects; + private Swagger swaggerFunders; + private Swagger swaggerCommunities; + private Swagger swaggerContexts; + private Swagger swaggerInfo; + + private Vocabularies vocabularies; + + public static class Datasource { + // MONGODB + private String mongoHost; + private int mongoPort; + private String mongoCollectionName; + private String mongoDbName; + private int mongoConnectionsPerHost; + private int mongoQueryLimit; + + public String getMongoHost() { + return mongoHost; + } + + public void setMongoHost(final String mongoHost) { + this.mongoHost = mongoHost; + } + + public int getMongoPort() { + return mongoPort; + } + + public void setMongoPort(final int mongoPort) { + this.mongoPort = mongoPort; + } + + public String getMongoCollectionName() { + return mongoCollectionName; + } + + public void setMongoCollectionName(final String mongoCollectionName) { + this.mongoCollectionName = mongoCollectionName; + } + + public String getMongoDbName() { + return mongoDbName; + } + + public void setMongoDbName(final String mongoDbName) { + this.mongoDbName = mongoDbName; + } + + public int getMongoConnectionsPerHost() { + return mongoConnectionsPerHost; + } + + public void setMongoConnectionsPerHost(final int mongoConnectionsPerHost) { + this.mongoConnectionsPerHost = mongoConnectionsPerHost; + } + + public int getMongoQueryLimit() { + return mongoQueryLimit; + } + + public void setMongoQueryLimit(final int mongoQueryLimit) { + this.mongoQueryLimit = mongoQueryLimit; + } + } + + public static class Project { + + private int flushSize; + private String tsvFields; + private Resource projectsFundingQueryTemplate; + private Resource dspaceTemplate; + private Resource dspaceHeadTemplate; + private Resource dspaceTailTemplate; + private Resource eprintsTemplate; + + public int getFlushSize() { + return flushSize; + } + + public void setFlushSize(final int flushSize) { + this.flushSize = flushSize; + } + + public String getTsvFields() { + return tsvFields; + } + + public void setTsvFields(final String tsvFields) { + this.tsvFields = tsvFields; + } + + public Resource getProjectsFundingQueryTemplate() { + return projectsFundingQueryTemplate; + } + + public void setProjectsFundingQueryTemplate(final Resource projectsFundingQueryTemplate) { + this.projectsFundingQueryTemplate = projectsFundingQueryTemplate; + } + + public Resource getDspaceTemplate() { + return dspaceTemplate; + } + + public void setDspaceTemplate(final Resource dspaceTemplate) { + this.dspaceTemplate = dspaceTemplate; + } + + public Resource getDspaceHeadTemplate() { + return dspaceHeadTemplate; + } + + public void setDspaceHeadTemplate(final Resource dspaceHeadTemplate) { + this.dspaceHeadTemplate = dspaceHeadTemplate; + } + + public Resource getDspaceTailTemplate() { + return dspaceTailTemplate; + } + + public void setDspaceTailTemplate(final Resource dspaceTailTemplate) { + this.dspaceTailTemplate = dspaceTailTemplate; + } + + public Resource getEprintsTemplate() { + return eprintsTemplate; + } + + public void setEprintsTemplate(final Resource eprintsTemplate) { + this.eprintsTemplate = eprintsTemplate; + } + } + + public static class Jdbc { + + // JDBC + @Value("${spring.datasource.driverClassName}") + private String driverClassName; + + private String url; + private String user; + private String pwd; + private int minIdle; + private int maxidle; + private int maxRows; + + public String getDriverClassName() { + return driverClassName; + } + + public String getUrl() { + return url; + } + + public void setUrl(final String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(final String user) { + this.user = user; + } + + public String getPwd() { + return pwd; + } + + public void setPwd(final String pwd) { + this.pwd = pwd; + } + + public int getMinIdle() { + return minIdle; + } + + public void setMinIdle(final int minIdle) { + this.minIdle = minIdle; + } + + public int getMaxidle() { + return maxidle; + } + + public void setMaxidle(final int maxidle) { + this.maxidle = maxidle; + } + + public int getMaxRows() { + return maxRows; + } + + public void setMaxRows(final int maxRows) { + this.maxRows = maxRows; + } + } + + public static class Swagger { + private String apiTitle; + private String apiDescription; + private String apiLicense; + private String apiLicenseUrl; + private String apiContactName; + private String apiContactUrl; + private String apiContactEmail; + + public String getApiTitle() { + return apiTitle; + } + + public void setApiTitle(final String apiTitle) { + this.apiTitle = apiTitle; + } + + public String getApiDescription() { + return apiDescription; + } + + public void setApiDescription(final String apiDescription) { + this.apiDescription = apiDescription; + } + + public String getApiLicense() { + return apiLicense; + } + + public void setApiLicense(final String apiLicense) { + this.apiLicense = apiLicense; + } + + public String getApiLicenseUrl() { + return apiLicenseUrl; + } + + public void setApiLicenseUrl(final String apiLicenseUrl) { + this.apiLicenseUrl = apiLicenseUrl; + } + + public String getApiContactName() { + return apiContactName; + } + + public void setApiContactName(final String apiContactName) { + this.apiContactName = apiContactName; + } + + public String getApiContactUrl() { + return apiContactUrl; + } + + public void setApiContactUrl(final String apiContactUrl) { + this.apiContactUrl = apiContactUrl; + } + + public String getApiContactEmail() { + return apiContactEmail; + } + + public void setApiContactEmail(final String apiContactEmail) { + this.apiContactEmail = apiContactEmail; + } + } + + public static class Vocabularies { + + private String baseUrl; + + private String countriesEndpoint; + + private String datasourceTypologiesEndpoint; + + public String getCountriesEndpoint() { + return countriesEndpoint; + } + + public void setCountriesEndpoint(final String countriesEndpoint) { + this.countriesEndpoint = countriesEndpoint; + } + + public String getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(final String baseUrl) { + this.baseUrl = baseUrl; + } + + public String getDatasourceTypologiesEndpoint() { + return datasourceTypologiesEndpoint; + } + + public void setDatasourceTypologiesEndpoint(final String datasourceTypologiesEndpoint) { + this.datasourceTypologiesEndpoint = datasourceTypologiesEndpoint; + } + } + + public ClassPathResource getFindSolrIndexUrl() { + return findSolrIndexUrl; + } + + public void setFindSolrIndexUrl(final ClassPathResource findSolrIndexUrl) { + this.findSolrIndexUrl = findSolrIndexUrl; + } + + public ClassPathResource getFindIndexDsInfo() { + return findIndexDsInfo; + } + + public ClassPathResource getFindObjectStore() { + return findObjectStore; + } + + public void setFindObjectStore(final ClassPathResource findObjectStore) { + this.findObjectStore = findObjectStore; + } + + public void setFindIndexDsInfo(final ClassPathResource findIndexDsInfo) { + this.findIndexDsInfo = findIndexDsInfo; + } + + public ClassPathResource getFindFunderContexts() { + return findFunderContexts; + } + + public void setFindFunderContexts(final ClassPathResource findFunderContexts) { + this.findFunderContexts = findFunderContexts; + } + + public ClassPathResource getFindCommunityContexts() { + return findCommunityContexts; + } + + public ClassPathResource getFindContextProfiles() { + return findContextProfiles; + } + + public ClassPathResource getFindContextProfilesByType() { + return findContextProfilesByType; + } + + public void setFindContextProfiles(final ClassPathResource findContextProfiles) { + this.findContextProfiles = findContextProfiles; + } + + public void setFindContextProfilesByType(ClassPathResource findContextProfilesByType) { + this.findContextProfilesByType = findContextProfilesByType; + } + + public void setFindCommunityContexts(final ClassPathResource findCommunityContexts) { + this.findCommunityContexts = findCommunityContexts; + } + + public ClassPathResource getGetRepoProfile() { + return getRepoProfile; + } + + public void setGetRepoProfile(final ClassPathResource getRepoProfile) { + this.getRepoProfile = getRepoProfile; + } + + public String getContentLoadQuery() { + return contentLoadQuery; + } + + public void setContentLoadQuery(String contentLoadQuery) { + this.contentLoadQuery = contentLoadQuery; + } + + public String getIsLookupUrl() { + return isLookupUrl; + } + + public void setIsLookupUrl(final String isLookupUrl) { + this.isLookupUrl = isLookupUrl; + } + + public String getObjectStoreServiceUrl() { + return objectStoreServiceUrl; + } + + public void setObjectStoreServiceUrl(final String objectStoreServiceUrl) { + this.objectStoreServiceUrl = objectStoreServiceUrl; + } + + public String getIsRegistryServiceUrl() { + return isRegistryServiceUrl; + } + + public void setIsRegistryServiceUrl(final String isRegistryServiceUrl) { + this.isRegistryServiceUrl = isRegistryServiceUrl; + } + + public int getRequestWorkers() { + return requestWorkers; + } + + public void setRequestWorkers(final int requestWorkers) { + this.requestWorkers = requestWorkers; + } + + public int getRequestTimeout() { + return requestTimeout; + } + + public void setRequestTimeout(final int requestTimeout) { + this.requestTimeout = requestTimeout; + } + + public int getCxfClientConnectTimeout() { + return cxfClientConnectTimeout; + } + + public void setCxfClientConnectTimeout(int cxfClientConnectTimeout) { + this.cxfClientConnectTimeout = cxfClientConnectTimeout; + } + + public int getCxfClientReceiveTimeout() { + return cxfClientReceiveTimeout; + } + + public void setCxfClientReceiveTimeout(int cxfClientReceiveTimeout) { + this.cxfClientReceiveTimeout = cxfClientReceiveTimeout; + } + + public Datasource getDatasource() { + return datasource; + } + + public void setDatasource(final Datasource datasource) { + this.datasource = datasource; + } + + public Project getProject() { + return project; + } + + public void setProject(final Project project) { + this.project = project; + } + + public Jdbc getJdbc() { + return jdbc; + } + + public void setJdbc(final Jdbc jdbc) { + this.jdbc = jdbc; + } + + public Swagger getSwaggerDsm() { + return swaggerDsm; + } + + public void setSwaggerDsm(final Swagger swaggerDsm) { + this.swaggerDsm = swaggerDsm; + } + + public Swagger getSwaggerProjects() { + return swaggerProjects; + } + + public void setSwaggerProjects(final Swagger swaggerProjects) { + this.swaggerProjects = swaggerProjects; + } + + public Swagger getSwaggerFunders() { + return swaggerFunders; + } + + public void setSwaggerFunders(final Swagger swaggerFunders) { + this.swaggerFunders = swaggerFunders; + } + + public Swagger getSwaggerCommunities() { + return swaggerCommunities; + } + + public void setSwaggerCommunities(final Swagger swaggerCommunities) { + this.swaggerCommunities = swaggerCommunities; + } + + public Swagger getSwaggerContexts() { + return swaggerContexts; + } + + public void setSwaggerContexts(final Swagger swaggerContexts) { + this.swaggerContexts = swaggerContexts; + } + + public Swagger getSwaggerInfo() { return swaggerInfo; } + + public void setSwaggerInfo(Swagger swaggerInfo) { this.swaggerInfo = swaggerInfo; } + + public Vocabularies getVocabularies() { + return vocabularies; + } + + public void setVocabularies(final Vocabularies vocabularies) { + this.vocabularies = vocabularies; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java new file mode 100644 index 00000000..b53bbbc4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/SwaggerController.java @@ -0,0 +1,16 @@ +package eu.dnetlib; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; + +@Controller +public class SwaggerController { + + @RequestMapping(value = { + "/", "/docs", "swagger-ui.html" + }) + public String index() { + return "redirect:swagger-ui/"; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java new file mode 100644 index 00000000..c11e15f0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java @@ -0,0 +1,100 @@ +package eu.dnetlib.openaire.common; + +import java.util.List; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * Created by claudio on 18/07/2017. + */ +public abstract class AbstractExporterController { + + private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM + + @ResponseBody + @ExceptionHandler({DsmException.class}) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public ErrorMessage handleDSMException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler(DsmForbiddenException.class) + @ResponseStatus(value = HttpStatus.FORBIDDEN) + public ErrorMessage handleForbiddenException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler({DsmNotFoundException.class}) + @ResponseStatus(value = HttpStatus.NOT_FOUND) + public ErrorMessage handleNotFoundException(final Exception e) { + return _handleError(e); + } + + @ResponseBody + @ExceptionHandler(MethodArgumentNotValidException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + public List processValidationError(final MethodArgumentNotValidException e) { + return e.getBindingResult() + .getFieldErrors().stream() + .map(fe -> new ErrorMessage( + String.format("field '%s'", fe.getField()), + String.format("rejected value '%s'", fe.getRejectedValue()), + fe.getDefaultMessage())) + .collect(Collectors.toList()); + } + + private ErrorMessage _handleError(final Exception e) { + log.error(e); + if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) { + return null; //socket is closed, cannot return any response + } else { + return new ErrorMessage(e); + } + } + + @JsonAutoDetect + public class ErrorMessage { + + private final String message; + private final String details; + private final String stacktrace; + + public ErrorMessage(final Exception e) { + this(e.getMessage(), "", ExceptionUtils.getStackTrace(e)); + } + + public ErrorMessage(final String message, final String details, final String stacktrace) { + this.message = message; + this.details = details; + this.stacktrace = stacktrace; + } + + public String getMessage() { + return this.message; + } + + public String getStacktrace() { + return this.stacktrace; + } + + public String getDetails() { + return details; + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java new file mode 100644 index 00000000..9f7d04e3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ConverterTextArray.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.common; + +import java.sql.Array; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import javax.sql.DataSource; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +/** + * Created by claudio on 05/07/2017. + */ +@Converter +public class ConverterTextArray implements AttributeConverter, Array>, ApplicationContextAware { + + private ApplicationContext applicationContext; + + @Override + public Array convertToDatabaseColumn(List attribute) { + + final Map datasources = applicationContext.getBeansOfType(DataSource.class); + DataSource source = datasources.values().stream().findFirst().get(); + + try { + Connection conn = source.getConnection(); + return conn.createArrayOf("text", attribute.toArray()); + + } catch (SQLException e) { + e.printStackTrace(); + } + + return null; + + } + + @Override + public List convertToEntityAttribute(Array dbData) { + try { + return Arrays.stream((Object[]) dbData.getArray()).map(d -> (String) d).collect(Collectors.toList()); + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java new file mode 100644 index 00000000..29e4e38a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ExporterConstants.java @@ -0,0 +1,30 @@ +package eu.dnetlib.openaire.common; + +public class ExporterConstants { + + /* + * Tags used to group the operations on the swagger UI + */ + public final static String C = "Community"; + public final static String C_CP = "Community content providers"; + public final static String C_PJ = "Community projects"; + public final static String C_ZC = "Community Zenodo Communities"; + public final static String C_O = "Community Organizations"; + + public final static String DS = "Datasource"; + public final static String API = "Interface"; + public final static String R = "Read"; + public final static String W = "Write"; + + public final static String D = "Deprecated"; + public final static String M = "Management"; + + public final static String DSPACE = "DSpace"; + public final static String EPRINT = "EPrints"; + public final static String TSV = "TSV"; + public final static String STREAMING = "Streaming"; + + public static final String OAI = "oai"; + public static final String SET = "set"; + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java new file mode 100644 index 00000000..51a4d738 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/GenericArrayUserType.java @@ -0,0 +1,106 @@ +package eu.dnetlib.openaire.common; + +import java.io.Serializable; +import java.sql.*; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.usertype.UserType; + +/** + * Created by claudio on 05/07/2017. + */ +public class GenericArrayUserType implements UserType { + + protected static final int[] SQL_TYPES = { Types.ARRAY }; + + private Class typeParameterClass; + + @Override + public Object assemble(Serializable cached, Object owner) throws HibernateException { + return this.deepCopy(cached); + } + + @Override + public Object deepCopy(Object value) throws HibernateException { + return value; + } + + @SuppressWarnings("unchecked") + @Override + public Serializable disassemble(Object value) throws HibernateException { + return (T) this.deepCopy(value); + } + + @Override + public boolean equals(Object x, Object y) throws HibernateException { + + if (x == null) { + return y == null; + } + return x.equals(y); + } + + @Override + public int hashCode(Object x) throws HibernateException { + return x.hashCode(); + } + + @Override + public Object nullSafeGet(final ResultSet resultSet, + final String[] names, + final SharedSessionContractImplementor sharedSessionContractImplementor, + final Object o) + throws HibernateException, SQLException { + if (resultSet.wasNull()) { + return null; + } + if (resultSet.getArray(names[0]) == null) { + return new Integer[0]; + } + + Array array = resultSet.getArray(names[0]); + @SuppressWarnings("unchecked") + T javaArray = (T) array.getArray(); + return javaArray; + } + + @Override + public void nullSafeSet(final PreparedStatement statement, + final Object value, + final int index, + final SharedSessionContractImplementor session) + throws HibernateException, SQLException { + Connection connection = statement.getConnection(); + if (value == null) { + statement.setNull(index, SQL_TYPES[0]); + } else { + @SuppressWarnings("unchecked") + T castObject = (T) value; + Array array = connection.createArrayOf("integer", (Object[]) castObject); + statement.setArray(index, array); + } + } + + @Override + public boolean isMutable() { + return true; + } + + @Override + public Object replace(Object original, Object target, Object owner) throws HibernateException { + return original; + } + + @Override + public Class returnedClass() { + return typeParameterClass; + } + + @Override + public int[] sqlTypes() { + return new int[] { Types.ARRAY }; + } + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java new file mode 100644 index 00000000..01752de5 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java @@ -0,0 +1,48 @@ +package eu.dnetlib.openaire.common; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; + +public interface ISClient { + + IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException; + + String getObjectStoreId(String dsId) throws DsmException; + + Map getFunderContextMap() throws IOException; + + Map getCommunityContextMap() throws IOException; + + Map getContextMap(final List type) throws IOException; + + void updateContextParam(String id, String name, String value); + + void updateContextAttribute(String id, String name, String value); + + void addConcept(String id, String categoryId, String data); + + void removeConcept(String id, String categoryId, String conceptId); + + void dropCache(); + + /** + * + * @param id + * id of the concept to be updated (i.e. ni::projects::2) + * @param name + * name of the attribute to be updated + * @param value + * new value for the attribute + */ + void updateConceptAttribute(String id, String name, String value); + + void updateConceptParam(String id, String name, String value); + + void updateConceptParamNoEscape(String id, String name, String value); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java new file mode 100644 index 00000000..3bb72c60 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java @@ -0,0 +1,297 @@ +package eu.dnetlib.openaire.common; + +import static eu.dnetlib.openaire.common.Utils.escape; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.core.io.ClassPathResource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.google.common.collect.Lists; +import com.google.common.escape.Escaper; +import com.google.common.xml.XmlEscapers; + +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.ContextMappingUtils; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; + +/** + * Created by claudio on 20/10/2016. + */ +@Component +public class ISClientImpl implements ISClient { + + private static final Log log = LogFactory.getLog(ISClientImpl.class); + + @Autowired + private DnetOpenaireExporterProperties config; + + @Autowired + private ISLookUpService isLookUpService; + + @Override + @Cacheable("indexdsinfo-cache") + public IndexDsInfo calculateCurrentIndexDsInfo() throws DsmException { + log.warn("calculateCurrentIndexDsInfo(): not using cache"); + final String[] arr; + try { + arr = _isLookUp(_getQuery(config.getFindIndexDsInfo())).split("@@@"); + return new IndexDsInfo( + _isLookUp(_getQuery(config.getFindSolrIndexUrl())), + arr[0].trim(), arr[1].trim(), arr[2].trim()); + } catch (IOException | ISLookUpException e) { + throw new DsmException("unable fetch index DS information from IS"); + } + } + + @Override + @Cacheable("objectstoreid-cache") + public String getObjectStoreId(final String dsId) throws DsmException { + log.warn(String.format("getObjectStoreId(%s): not using cache", dsId)); + try { + final String xqueryTemplate = _getQuery(config.getFindObjectStore()); + return _isLookUp(String.format(xqueryTemplate, dsId)); + } catch (IOException | ISLookUpException e) { + throw new DsmException("unble to find objectstore for ds " + dsId); + } + } + + @Override + @Cacheable("context-cache-funder") + public Map getFunderContextMap() throws IOException { + return _processContext(_getQuery(config.getFindFunderContexts())); + } + + @Override + @Cacheable("context-cache-community") + public Map getCommunityContextMap() throws IOException { + return _processContext(_getQuery(config.getFindCommunityContexts())); + } + + @Override + @Cacheable("context-cache") + public Map getContextMap(final List type) throws IOException { + if (Objects.isNull(type) || type.isEmpty()) { + return _processContext(_getQuery(config.getFindContextProfiles())); + } else { + final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType()); + + final String xquery = String.format(xqueryTemplate, type.stream() + .map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t)) + .collect(Collectors.joining(" or "))); + + return _processContext(xquery); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateContextParam(final String id, final String name, final String value) { + try { + _quickSeachProfile(getXQuery(id, name, value)); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update context param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateContextAttribute(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + try { + _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update context attribute [id: %s, name: %s, data: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void addConcept(final String id, final String categoryId, final String data) { + try { + _quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/category[./@id = '%s']", data, id, categoryId)); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable add concept [id: %s, categoryId: %s, data: %s]", id, categoryId, data), e); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void removeConcept(final String id, final String categoryId, final String conceptId) { + try { + _quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']" + + "/category[./@id = '%s']/concept[./@id = '%s'] " + + "return update delete $concept", id, categoryId, conceptId)); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable remove concept [id: %s, categoryId: %s, conceptId: %s]", id, categoryId, conceptId), e); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-community", "context-cache-funder" + }, allEntries = true) + public void updateConceptAttribute(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + try { + _quickSeachProfile(String.format("update value collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context/category/concept[./@id = '%s']/@%s with '%s'", id, name, escape(esc, value))); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept attribute [id: %s, name: %s, value: %s]", id, name, value), e); + } + + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateConceptParam(final String id, final String name, final String value) { + try { + _quickSeachProfile(getConceptXQuery(id, name, value)); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + @Override + @CacheEvict(value = { + "context-cache", "context-cache-funder" + }, allEntries = true) + public void updateConceptParamNoEscape(final String id, final String name, final String value) { + try { + _quickSeachProfile(getConceptXQueryNoEscape(id, name, value)); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException(String.format("unable update concept param [id: %s, name: %s, value: %s]", id, name, value), e); + } + } + + /// HELPERS + + private String getXQuery(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlContentEscaper(); + if (StringUtils.isNotBlank(value)) { + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); + } else { + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" + + "/RESOURCE_PROFILE/BODY/CONFIGURATION/context[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private String getConceptXQuery(final String id, final String name, final String value) { + final Escaper esc = XmlEscapers.xmlContentEscaper(); + if (StringUtils.isNotBlank(value)) { + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, escape(esc, value)); + } else { + return String + .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private String getConceptXQueryNoEscape(final String id, final String name, final String value) { + + if (StringUtils.isNotBlank(value)) { + return String.format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//" + + "concept[./@id = '%s']/param[./@name = '%s'] with %s", id, name, name, value); + } else { + return String + .format("update replace collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')//concept[./@id = '%s']/param[./@name = '%s'] with ", id, name, name); + } + } + + private Map _processContext(final String xquery) throws IOException { + return _processContext(new LinkedBlockingQueue<>(), xquery); + } + + private Map _processContext(final Queue errors, final String xquery) throws IOException { + try { + return getContextProfiles(errors, xquery).stream() + .filter(StringUtils::isNotBlank) + .map(s -> ContextMappingUtils.parseContext(s, errors)) + .collect(Collectors.toMap(Context::getId, Function.identity(), (c1, c2) -> { + log.warn(String.format("found duplicate context profile '%s'", c1.getId())); + return c1; + })); + } finally { + if (!errors.isEmpty()) { + log.error(errors); + errors.forEach(Throwable::printStackTrace); + } + } + } + + private List getContextProfiles(final Queue errors, final String xquery) throws IOException { + log.warn("getContextProfiles(): not using cache"); + try { + return _quickSeachProfile(xquery); + } catch (final ISLookUpException e) { + throw new DsmRuntimeException("unable to get context profiles", e); + } + } + + private String _getQuery(final ClassPathResource resource) throws IOException { + return IOUtils.toString(resource.getInputStream(), Charset.defaultCharset()); + } + + private String _isLookUp(final String xquery) throws ISLookUpException { + log.debug(String.format("running xquery:\n%s", xquery)); + // log.debug(String.format("query result: %s", res)); + return isLookUpService.getResourceProfileByQuery(xquery); + } + + private List _quickSeachProfile(final String xquery) throws ISLookUpException { + final List res = Lists.newArrayList(); + + log.debug(String.format("running xquery:\n%s", xquery)); + final List list = isLookUpService.quickSearchProfile(xquery); + if (list != null) { + res.addAll(list); + } + log.debug(String.format("query result size: %s", res.size())); + return res; + } + + @Override + @CacheEvict(cacheNames = { + "context-cache", "indexdsinfo-cache", "objectstoreid-cache" + }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager IS cache"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java new file mode 100644 index 00000000..532c1890 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/OperationManager.java @@ -0,0 +1,60 @@ +package eu.dnetlib.openaire.common; + +import java.util.List; +import java.util.concurrent.*; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.stereotype.Component; + +@Component +public class OperationManager { + + private static final Log log = LogFactory.getLog(OperationManager.class); + + private static final long SLEEP_TIME = 1000; + + private static final int Q_SIZE = 100; + + private static final int POOL_SIZE = 5; + + private final BlockingQueue ops = new ArrayBlockingQueue<>(Q_SIZE); + + private ExecutorService executor; + + @PostConstruct + public void init() { + executor = getExecutor(); + } + + public int dropAll() { + final List lostOperations = executor.shutdownNow(); + log.warn(String.format("discarding %s operations", lostOperations.size())); + executor = getExecutor(); + return lostOperations.size(); + } + + public int getOpSize() { + return ops.size(); + } + + public void addOperation(final Runnable op) { + executor.execute(op); + } + + @PreDestroy + public void tearDown() throws InterruptedException { + executor.shutdown(); + final boolean done = executor.awaitTermination(SLEEP_TIME, TimeUnit.MILLISECONDS); + log.debug(String.format("All operations were completed so far? %s", done)); + } + + // HELPERS + + private ThreadPoolExecutor getExecutor() { + return new ThreadPoolExecutor(POOL_SIZE, POOL_SIZE,0L, TimeUnit.MILLISECONDS, ops); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java new file mode 100644 index 00000000..51c35b25 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/RFC3339DateFormat.java @@ -0,0 +1,63 @@ +package eu.dnetlib.openaire.common; + +import java.text.FieldPosition; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.TimeZone; + +import com.fasterxml.jackson.databind.util.StdDateFormat; + +public class RFC3339DateFormat extends StdDateFormat { + + /** + * + */ + private static final long serialVersionUID = 8174507696046505992L; + + private static final TimeZone TIMEZONE_Z = TimeZone.getTimeZone("UTC"); + + // Same as ISO8601DateFormat but serializing milliseconds. + @Override + public StringBuffer format(final Date date, final StringBuffer toAppendTo, final FieldPosition fieldPosition) { + final String value = format(date, true, TIMEZONE_Z, Locale.US); + toAppendTo.append(value); + return toAppendTo; + } + + /** + * Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] + * + * @param date + * the date to format + * @param millis + * true to include millis precision otherwise false + * @param tz + * timezone to use for the formatting (UTC will produce 'Z') + * @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] + */ + private static String format(final Date date, final boolean millis, final TimeZone tz, final Locale loc) { + final Calendar calendar = new GregorianCalendar(tz, loc); + calendar.setTime(date); + + // estimate capacity of buffer as close as we can (yeah, that's pedantic ;) + final StringBuilder sb = new StringBuilder(30); + sb.append(String.format("%04d-%02d-%02dT%02d:%02d:%02d", calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar + .get(Calendar.DAY_OF_MONTH), calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND))); + if (millis) { + sb.append(String.format(".%03d", calendar.get(Calendar.MILLISECOND))); + } + + final int offset = tz.getOffset(calendar.getTimeInMillis()); + if (offset != 0) { + final int hours = Math.abs(offset / (60 * 1000) / 60); + final int minutes = Math.abs(offset / (60 * 1000) % 60); + sb.append(String.format("%c%02d:%02d", offset < 0 ? '-' : '+', hours, minutes)); + } else { + sb.append('Z'); + } + return sb.toString(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java new file mode 100644 index 00000000..600e29d6 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/Utils.java @@ -0,0 +1,22 @@ +package eu.dnetlib.openaire.common; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import com.google.common.escape.Escaper; +import org.apache.commons.lang3.StringUtils; + +public class Utils { + + public static Stream stream(Iterator iterator) { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); + } + + public static String escape(final Escaper esc, final String value) { + return StringUtils.isNotBlank(value) ? esc.escape(value) : ""; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java new file mode 100644 index 00000000..e653b09a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java @@ -0,0 +1,304 @@ +package eu.dnetlib.openaire.community; + +import java.util.List; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +import static eu.dnetlib.openaire.common.ExporterConstants.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Communities API", description = "the OpenAIRE Community API") +public class CommunityApiController { + + @Autowired + private CommunityApiCore communityApiCore; + + + @RequestMapping(value = "/community/communities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get all community profiles", + notes = "get all community profiles", + tags = { C, R }, + response = CommunitySummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunitySummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List listCommunities() throws CommunityException { + return communityApiCore.listCommunities(); + } + + @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get community profile", + notes = "get community profile", + tags = { C, R }, + response = CommunityDetails.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityDetails.class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails getCommunity(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunity(id); + } + + @RequestMapping(value = "/community/{id}", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "update community details", + notes = "update community details", + tags = { C, R }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void setCommunity( + @PathVariable final String id, + @RequestBody CommunityWritableProperties properties) throws CommunityException, CommunityNotFoundException { + + communityApiCore.setCommunity(id, properties); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get community projects", + notes = "get community projects", + tags = { C_PJ, R }, + response = CommunityProject[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityProject[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityProjects(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityProjects(id); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a project to the community", + notes = "associate a project to the community", + tags = { C_PJ, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityProject addCommunityProject( + @PathVariable final String id, + @RequestBody final CommunityProject project) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityProject(id, project); + } + + @RequestMapping(value = "/community/{id}/projects", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove a project from the community", + notes = "remove a project from the community", + tags = { C_PJ, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void deleteCommunityProject( + @PathVariable final String id, + @RequestBody final Integer projectId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityProject(id, projectId); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of content providers associated to a given community", + notes = "get the list of content providers associated to a given community", + tags = { C_CP, R }, + response = CommunityContentprovider[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityContentproviders(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityContentproviders(id); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a content provider to the community", + notes = "associate a content provider to the community", + tags = { C_CP, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityContentprovider addCommunityContentprovider( + @PathVariable final String id, + @RequestBody final CommunityContentprovider contentprovider) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityContentprovider(id, contentprovider); + } + + @RequestMapping(value = "/community/{id}/contentproviders", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove the association between a content provider and the community", + notes = "remove the association between a content provider and the community", + tags = { C_CP, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityContentprovider( + @PathVariable final String id, + @RequestBody final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityContentProvider(id, contentproviderId); + } + + //ADDING CODE FOR COMMUNITY ORGANIZATIONS + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of organizations for a given community", + notes = "get the list of organizations for a given community", + tags = { C_O, R }, + response = CommunityOrganization[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityContentprovider[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityOrganizations(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityOrganizations(id); + } + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate an organization to the community", + notes = "associate an organization to the community", + tags = { C_O, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityOrganization addCommunityOrganization( + @PathVariable final String id, + @RequestBody final CommunityOrganization organization) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityOrganization(id, organization); + } + + @RequestMapping(value = "/community/{id}/organizations", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove the association between an organization and the community", + notes = "remove the association between an organization and the community", + tags = { C_O, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityOrganization( + @PathVariable final String id, + @RequestBody final Integer organizationId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityOrganization(id, organizationId); + } + //********************** + + @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a subject to the community", + notes = "associate a subject to the community", + tags = { C, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails addCommunitySubjects( + @PathVariable final String id, + @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunitySubjects(id, subjects); + } + + @RequestMapping(value = "/community/{id}/subjects", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove subjects from a community", + notes = "remove subjects from a community", + tags = { C, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityDetails removeCommunitySubjects( + @PathVariable final String id, + @RequestBody final List subjects) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.removeCommunitySubjects(id, subjects); + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of Zenodo communities associated to a given community", + notes = "get the list of Zenodo communities associated to a given community", + tags = { C_ZC, R }, + response = CommunityZenodoCommunity[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CommunityZenodoCommunity[].class), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public List getCommunityZenodoCommunities(@PathVariable final String id) throws CommunityException, CommunityNotFoundException { + return communityApiCore.getCommunityZenodoCommunities(id); + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.POST) + @ApiOperation( + value = "associate a Zenodo community to the community", + notes = "associate a Zenodo community to the community", + tags = { C_ZC, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityZenodoCommunity addCommunityZenodoCommunity( + @PathVariable final String id, + @RequestBody final CommunityZenodoCommunity zenodocommunity) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.addCommunityZenodoCommunity(id, zenodocommunity); + + } + + @RequestMapping(value = "/community/{id}/zenodocommunities", produces = { "application/json" }, method = RequestMethod.DELETE) + @ApiOperation( + value = "remove a Zenodo community from a community", + notes = "remove a Zenodo community from a community", + tags = { C_ZC, W }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public void removeCommunityZenodoCommunity( + @PathVariable final String id, + @RequestBody final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { + + communityApiCore.removeCommunityZenodoCommunity(id, zenodoCommId); + + } + + + @RequestMapping(value = "/community/{zenodoId}/openairecommunities", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get the list of OpenAIRE communities associated to a given Zenodo community", + notes = "get the list of OpenAIRE communities associated to a given Zenodo community", + tags = { C_ZC, R }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 404, message = "not found", response = CommunityNotFoundException.class), + @ApiResponse(code = 500, message = "unexpected error", response = CommunityException.class) }) + public CommunityOpenAIRECommunities getOpenAireCommunities( + @PathVariable final String zenodoId) throws CommunityException, CommunityNotFoundException { + + return communityApiCore.getOpenAIRECommunities(zenodoId); + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java new file mode 100644 index 00000000..bc8c2c18 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiCore.java @@ -0,0 +1,368 @@ +package eu.dnetlib.openaire.community; + +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_OFFICIALNAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CCONTENTPROVIDER_SELCRITERIA; +import static eu.dnetlib.openaire.community.CommunityConstants.CLABEL; +import static eu.dnetlib.openaire.community.CommunityConstants.CONTENTPROVIDERS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_LOGOURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CORGANIZATION_WEBSITEURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SUBJECT; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_ACRONYM; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FULLNAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_FUNDER; +import static eu.dnetlib.openaire.community.CommunityConstants.CPROJECT_NUMBER; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_DESCRIPTION; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_LOGOURL; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_NAME; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_STATUS; +import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_ZENODOC; +import static eu.dnetlib.openaire.community.CommunityConstants.CSV_DELIMITER; +import static eu.dnetlib.openaire.community.CommunityConstants.ID_SEPARATOR; +import static eu.dnetlib.openaire.community.CommunityConstants.OPENAIRE_ID; +import static eu.dnetlib.openaire.community.CommunityConstants.ORGANIZATION_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.PROJECTS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityConstants.ZENODOCOMMUNITY_ID_SUFFIX; + +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.stereotype.Component; + +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import eu.dnetlib.openaire.common.ISClient; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true") +public class CommunityApiCore {// implements CommunityClient{ + + private static final Log log = LogFactory.getLog(CommunityApiCore.class); + + @Autowired + private CommunityClient cci; + + @Autowired + private ISClient isClient; + + @Autowired + private CommunityCommon cc; + + public List listCommunities() throws CommunityException { + return cc.listCommunities(); + + } + + public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { + return cc.getCommunity(id); + + } + + public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, CommunityNotFoundException { + + cc.getCommunity(id); // ensure the community exists. + + if (details.getShortName() != null) { + isClient.updateContextAttribute(id, CLABEL, details.getShortName()); + + } + if (details.getName() != null) { + isClient.updateContextParam(id, CSUMMARY_NAME, details.getName()); + + } + if (details.getDescription() != null) { + isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription()); + + } + if (details.getLogoUrl() != null) { + isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl()); + + } + if (details.getStatus() != null) { + isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name()); + + } + if (details.getSubjects() != null) { + isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects())); + + } + if (details.getMainZenodoCommunity() != null) { + isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity()); + } + + cc.updateCommunity(id, details); + } + + public List getCommunityProjects(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c)); + } + + public CommunityProject addCommunityProject(final String id, final CommunityProject project) throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, project.getCommunityId())) { + throw new CommunityException("parameters 'id' and project.communityId must be coherent"); + } + + final TreeMap projects = getCommunityProjectMap(id); + final String project_id = project.getId(); + + if (project_id != null && projects.keySet().contains(Integer.valueOf(project_id))) { + if (project.getName() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FULLNAME, project.getName()); + + } + if (project.getAcronym() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_ACRONYM, project.getAcronym()); + + } + if (project.getOpenaireId() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, OPENAIRE_ID, project.getOpenaireId()); + + } + if (project.getFunder() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_FUNDER, project.getFunder()); + + } + if (project.getGrantId() != null) { + isClient.updateConceptParam(id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project_id, CPROJECT_NUMBER, project.getGrantId()); + + } + } else { + project.setId(nextId(projects != null && !projects.isEmpty() ? projects.lastKey() : 0)); + + isClient.addConcept(id, id + PROJECTS_ID_SUFFIX, CommunityMappingUtils.asProjectXML(id, project)); + + } + cc.updateProject(id, project); + return project; + } + + private String nextId(final Integer id) { + return String.valueOf(id + 1); + } + + public void removeCommunityProject(final String id, final Integer projectId) throws CommunityException, CommunityNotFoundException { + final Map projects = getCommunityProjectMap(id); + if (!projects.containsKey(projectId)) { + throw new CommunityNotFoundException(String.format("project '%s' doesn't exist within context '%s'", projectId, id)); + } + isClient.removeConcept(id, id + PROJECTS_ID_SUFFIX, id + PROJECTS_ID_SUFFIX + ID_SEPARATOR + projectId); + cc.removeFromCategory(id, PROJECTS_ID_SUFFIX, String.valueOf(projectId)); + } + + public List getCommunityContentproviders(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); // ensure the community exists. + return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c)); + } + + public CommunityContentprovider addCommunityContentprovider(final String id, final CommunityContentprovider cp) + throws CommunityException, CommunityNotFoundException { + log.info("content provider to add " + cp.toString()); + if (!StringUtils.equalsIgnoreCase(id, cp.getCommunityId())) { throw new CommunityException("parameters 'id' and cp.communityId must be coherent"); } + + final TreeMap cps = getCommunityContentproviderMap(id); + final String concept_id = cp.getId(); + if (concept_id != null && cps.keySet().contains(Integer.valueOf(concept_id))) { + if (cp.getName() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_NAME, cp.getName()); + } + if (cp.getOfficialname() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_OFFICIALNAME, cp.getOfficialname()); + } + if (cp.getOpenaireId() != null) { + isClient.updateConceptParam(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, OPENAIRE_ID, cp.getOpenaireId()); + } + if (cp.getSelectioncriteria() != null) { + isClient.updateConceptParamNoEscape(id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + concept_id, CCONTENTPROVIDER_SELCRITERIA, cp.toXML()); + } + } else { + log.info("adding new concept for community " + id); + cp.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + + isClient.addConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, CommunityMappingUtils.asContentProviderXML(id, cp)); + } + + cc.updateDatasource(id, cp); + return cp; + } + + public void removeCommunityContentProvider(final String id, final Integer contentproviderId) throws CommunityException, CommunityNotFoundException { + final Map providers = getCommunityContentproviderMap(id); + if (!providers.containsKey(contentproviderId)) { + throw new CommunityNotFoundException(String.format("content provider '%s' doesn't exist within context '%s'", contentproviderId, id)); + } + isClient.removeConcept(id, id + CONTENTPROVIDERS_ID_SUFFIX, id + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + contentproviderId); + cc.removeFromCategory(id, CONTENTPROVIDERS_ID_SUFFIX, String.valueOf(contentproviderId)); + } + + public void removeCommunityOrganization(final String id, final Integer organizationId) throws CommunityException, CommunityNotFoundException { + final Map organizations = getCommunityOrganizationMap(id); + if (!organizations.containsKey(organizationId)) { + throw new CommunityNotFoundException(String.format("organization '%s' doesn't exist within context '%s'", organizationId, id)); + } + isClient.removeConcept(id, id + ORGANIZATION_ID_SUFFIX, id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organizationId); + cc.removeFromCategory(id, ORGANIZATION_ID_SUFFIX, String.valueOf(organizationId)); + } + + public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { + + return cc.getCommunityZenodoCommunities(id); + } + + public List getCommunityOrganizations(final String id) throws CommunityException, CommunityNotFoundException { + cc.getCommunity(id); + return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c)); + } + + public CommunityDetails addCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.addAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + public CommunityDetails removeCommunitySubjects(final String id, final List subjects) throws CommunityException, CommunityNotFoundException { + + final CommunityDetails cd = new CommunityDetails(); + + final Set current = Sets.newHashSet(cc.getCommunity(id).getSubjects()); + + current.removeAll(subjects); + + cd.setSubjects(Lists.newArrayList(current)); + + setCommunity(id, CommunityWritableProperties.fromDetails(cd)); + + return cd; + } + + @CacheEvict(value = "community-cache", allEntries = true) + public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, CommunityNotFoundException { + + final Map zcomms = getZenodoCommunityMap(id); + if (!zcomms.containsKey(zenodoCommId)) { + throw new CommunityNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id)); + } + isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId); + + cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId)); + + } + + @CacheEvict(value = "community-cache", allEntries = true) + public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc) + throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); } + if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); } + final TreeMap zcs = getZenodoCommunityMap(id); + + for (final CommunityZenodoCommunity czc : zcs.values()) { + if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); } + } + + zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0)); + + isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc)); + cc.updateZenodoCommunity(id, zc); + + return zc; + } + + public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, CommunityNotFoundException { + + if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) { + return new CommunityOpenAIRECommunities().setZenodoid(zenodoId) + .setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList())); + } + return new CommunityOpenAIRECommunities(); + + } + + // HELPERS + + private TreeMap getCommunityProjectMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityProjects(id).stream() + .collect(Collectors.toMap(p -> Integer.valueOf(p.getId()), Functions.identity(), (p1, p2) -> { + log.warn(String.format("duplicate project found: '%s'", p1.getId())); + return p2; + }, TreeMap::new)); + } + + private TreeMap getCommunityContentproviderMap(final String id) throws CommunityException, CommunityNotFoundException { + log.info("getting community content provider map"); + return getCommunityContentproviders(id).stream() + .collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> { + log.warn(String.format("duplicate content provider found: '%s'", cp1.getId())); + return cp2; + }, TreeMap::new)); + } + + private TreeMap getZenodoCommunityMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityZenodoCommunities(id).stream() + .collect(Collectors.toMap(cp -> Integer.valueOf(cp.getId()), Functions.identity(), (cp1, cp2) -> { + log.warn(String.format("duplicate Zenodo community found: '%s'", cp1.getId())); + return cp2; + }, TreeMap::new)); + } + + private TreeMap getCommunityOrganizationMap(final String id) throws CommunityException, CommunityNotFoundException { + return getCommunityOrganizations(id).stream() + .collect(Collectors.toMap(o -> Integer.valueOf(o.getId()), Functions.identity(), (o1, o2) -> { + log.warn(String.format("duplicate content provider found: '%s'", o1.getId())); + return o2; + }, TreeMap::new)); + } + + public CommunityOrganization addCommunityOrganization(final String id, final CommunityOrganization organization) + throws CommunityException, CommunityNotFoundException { + if (!StringUtils.equalsIgnoreCase(id, organization.getCommunityId())) { + throw new CommunityException("parameters 'id' and organization.communityId must be coherent"); + } + + final TreeMap cps = getCommunityOrganizationMap(id); + + final String organization_id = organization.getId(); + if (organization_id != null && cps.keySet().contains(Integer.valueOf(organization_id))) { + if (organization.getName() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_NAME, organization.getName()); + } + if (organization.getLogo_url() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_LOGOURL, Base64.getEncoder() + .encodeToString(organization.getLogo_url().getBytes())); + } + if (organization.getWebsite_url() != null) { + isClient.updateConceptParam(id + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization_id, CORGANIZATION_WEBSITEURL, Base64.getEncoder() + .encodeToString(organization.getWebsite_url().getBytes())); + } + } else { + organization.setId(nextId(!cps.isEmpty() ? cps.lastKey() : 0)); + isClient.addConcept(id, id + ORGANIZATION_ID_SUFFIX, CommunityMappingUtils.asOrganizationXML(id, organization)); + } + + cc.updateOrganization(id, organization); + return organization; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java new file mode 100644 index 00000000..dbc18fc5 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClient.java @@ -0,0 +1,12 @@ +package eu.dnetlib.openaire.community; + +import java.util.Map; +import java.util.Set; + +public interface CommunityClient { + + Map> getInverseZenodoCommunityMap() throws CommunityException, CommunityNotFoundException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java new file mode 100644 index 00000000..96ac59a6 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityClientImpl.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.community; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class CommunityClientImpl implements CommunityClient { + + private static final Log log = LogFactory.getLog(CommunityClient.class); + + @Autowired + private CommunityCommon communityCommon; + + @Override + @Cacheable("community-cache") + public Map> getInverseZenodoCommunityMap () throws CommunityException, CommunityNotFoundException { + log.info("Creating the data structure. Not using cache"); + final Map> inverseListMap = new HashMap<>(); + + final List communityList = communityCommon.listCommunities(); + + for(CommunitySummary cs :communityList){ + final String communityId = cs.getId(); + List czc = communityCommon.getCommunityZenodoCommunities(communityId); + for(CommunityZenodoCommunity zc:czc){ + final String zenodoId = zc.getZenodoid(); + if(!inverseListMap.containsKey(zenodoId)) { + inverseListMap.put(zc.getZenodoid(),new HashSet<>()); + } + inverseListMap.get(zc.getZenodoid()).add(communityId); + } + final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity(); + if(!inverseListMap.containsKey(zenodoMainCommunity)) { + inverseListMap.put(zenodoMainCommunity,new HashSet<>()); + } + + inverseListMap.get(zenodoMainCommunity).add(communityId); + } + return inverseListMap; + } + + + + @Override + @CacheEvict(cacheNames = { "community-cache", "context-cache-community"}, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache(){ + log.debug("dropped community cache"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java new file mode 100644 index 00000000..fde84333 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java @@ -0,0 +1,438 @@ +package eu.dnetlib.openaire.community; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import eu.dnetlib.openaire.common.ISClient; +import eu.dnetlib.openaire.context.Category; +import eu.dnetlib.openaire.context.Concept; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.Param; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static eu.dnetlib.openaire.community.CommunityConstants.*; + +@Component +public class CommunityCommon { + + @Autowired + private ISClient isClient; + + public Map getContextMap() throws CommunityException { + try { + return isClient.getCommunityContextMap(); + } catch (IOException e) { + throw new CommunityException(e); + } + } + + public List listCommunities() throws CommunityException { + return getContextMap().values().stream() + .filter(context -> !communityBlackList.contains(context.getId())) + .map(CommunityMappingUtils::asCommunitySummary) + .collect(Collectors.toList()); + } + + + public List getCommunityInfo(final String id, final String idSuffix, final Function mapping) throws CommunityException { + final Map contextMap = getContextMap(); + final Context context = contextMap.get(id); + if (context != null) { + final Map categories = context.getCategories(); + final Category category = categories.get(id + idSuffix); + if (category != null) { + return category.getConcepts().stream() + .map(mapping) + .collect(Collectors.toList()); + } + } + return Lists.newArrayList(); + } + + public CommunityDetails getCommunity(final String id) throws CommunityException, CommunityNotFoundException { + final Context context = getContextMap().get(id); + if (context == null || CommunityConstants.communityBlackList.contains(id)) { + throw new CommunityNotFoundException(String.format("community '%s' does not exist", id)); + } + return CommunityMappingUtils.asCommunityProfile(context); + } + + public List getCommunityZenodoCommunities(final String id) throws CommunityException, CommunityNotFoundException { + getCommunity(id); // ensure the community exists. + return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c)); + } + + + public void updateProject(String communityId, CommunityProject project) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category prj = context.getCategories().get(communityId + PROJECTS_ID_SUFFIX); + if (prj.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())){ + prj.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId())) { + if (project.getName() != null) { + + concept.getParams().replace(CPROJECT_FULLNAME, Arrays.asList(new Param() + .setName(CPROJECT_FULLNAME).setValue(project.getName()))); + } + if (project.getAcronym() != null) { + if(concept.getParams().keySet().contains(CPROJECT_ACRONYM)){ + concept.getParams().replace(CPROJECT_ACRONYM, Arrays.asList(new Param() + .setName(CPROJECT_ACRONYM).setValue(project.getAcronym()))); + } + else{ + concept.getParams().put(CPROJECT_ACRONYM, Arrays.asList(new Param() + .setName(CPROJECT_ACRONYM).setValue(project.getAcronym()))); + } + + } + if (project.getOpenaireId() != null) { + if(concept.getParams().keySet().contains(OPENAIRE_ID)){ + concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(project.getOpenaireId()))); + } + else{ + concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(project.getOpenaireId()))); + } + + } + if (project.getFunder() != null) { + concept.getParams().replace(CPROJECT_FUNDER, Arrays.asList(new Param() + .setName(CPROJECT_FUNDER).setValue(project.getFunder()))); + } + if (project.getGrantId() != null) { + concept.getParams().replace(CPROJECT_NUMBER, Arrays.asList(new Param() + .setName(CPROJECT_NUMBER).setValue(project.getGrantId()))); + + } + } + }); + } + else{ + Concept concept = new Concept(); + concept.setId(communityId + PROJECTS_ID_SUFFIX + ID_SEPARATOR + project.getId()); + concept.setClaim(false); + if(project.getAcronym() != null) + concept.setLabel(project.getAcronym()); + else + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if(project.getAcronym() != null){ + params.put(CPROJECT_ACRONYM, Arrays.asList(new Param().setName(CPROJECT_ACRONYM) + .setValue(project.getAcronym()))); + } + + if (project.getName() != null){ + params.put(CPROJECT_FULLNAME, Arrays.asList(new Param() + .setName(CPROJECT_FULLNAME) + .setValue(project.getName()) + )); + } + + if (project.getOpenaireId() != null){ + params.put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID) + .setValue(project.getOpenaireId()) + )); + } + + if(project.getFunder() != null){ + params.put(CPROJECT_FUNDER, Arrays.asList(new Param() + .setName(CPROJECT_FUNDER) + .setValue(project.getFunder()) + )); + } + + if (project.getGrantId()!=null){ + params.put(CPROJECT_NUMBER, Arrays.asList(new Param() + .setName(CPROJECT_NUMBER) + .setValue(project.getGrantId()) + )); + } + + concept.setParams(params); + prj.getConcepts().add(concept); + + + } + + } + + public void updateCommunity(String id, CommunityWritableProperties community) throws CommunityException { + final Context context = getContextMap().get(id); + + if(community.getShortName() != null) { + context.setLabel(community.getShortName()); + } + + if (community.getName() != null){ + context.getParams().replace(CSUMMARY_NAME, Arrays.asList(new Param() + .setValue(community.getName()).setName(CSUMMARY_NAME))); + } + if(community.getDescription() != null) { + context.getParams() + .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param() + .setName(CSUMMARY_DESCRIPTION).setValue(community.getDescription()))); + } + if(community.getLogoUrl() != null){ + context.getParams() + .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param() + .setName(CSUMMARY_LOGOURL).setValue(community.getLogoUrl()))); + + } + if (community.getStatus() != null) { + context.getParams() + .replace(CSUMMARY_STATUS, Arrays.asList(new Param() + .setName(CSUMMARY_STATUS).setValue(community.getStatus().name()))); + } + if (community.getSubjects() != null) { + context.getParams() + .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT) + .setValue(Joiner.on(CSV_DELIMITER) + .join(community.getSubjects())))); + } + if(community.getMainZenodoCommunity() != null){ + context.getParams() + .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param() + .setName(CSUMMARY_ZENODOC).setValue(community.getMainZenodoCommunity()))); + + } + + } + + + public void removeFromCategory(String communityId, String category, String conceptId) throws CommunityException { + Map cmap = getContextMap(); + Context context = cmap.get(communityId); + Map cat = context.getCategories(); + + List concepts = cat.get(communityId + category).getConcepts() + .stream().filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)).collect(Collectors.toList()); + + cat.get(communityId + category).setConcepts(concepts); + } + + public void updateDatasource(String communityId, CommunityContentprovider cp) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category dts = context.getCategories().get(communityId + CONTENTPROVIDERS_ID_SUFFIX); + if (dts.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())){ + dts.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId())) { + + + if (cp.getName() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_NAME)){ + concept.getParams().replace(CCONTENTPROVIDER_NAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_NAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_NAME).setValue(cp.getName()))); + } + + } + if (cp.getOfficialname() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_OFFICIALNAME)){ + concept.getParams().replace(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_OFFICIALNAME).setValue(cp.getOfficialname()))); + } + + } + + + if (cp.getOpenaireId() != null) { + if(concept.getParams().keySet().contains(OPENAIRE_ID)){ + concept.getParams().replace(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(cp.getOpenaireId()))); + } + else{ + concept.getParams().put(OPENAIRE_ID, Arrays.asList(new Param() + .setName(OPENAIRE_ID).setValue(cp.getOpenaireId()))); + } + + } + + if (cp.getSelectioncriteria() != null) { + if(concept.getParams().keySet().contains(CCONTENTPROVIDER_SELCRITERIA)){ + concept.getParams().replace(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson()))); + } + else{ + concept.getParams().put(CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param() + .setName(CCONTENTPROVIDER_SELCRITERIA).setValue(cp.toJson()))); + } + + } + } + }); + } + else{ + Concept concept = new Concept(); + concept.setId(communityId + CONTENTPROVIDERS_ID_SUFFIX + ID_SEPARATOR + cp.getId()); + concept.setClaim(false); + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if (cp.getName() != null) { + params.put( CCONTENTPROVIDER_NAME, Arrays.asList(new Param().setValue(cp.getName()).setName(CCONTENTPROVIDER_NAME))); + } + if(cp.getOfficialname()!= null){ + params.put( CCONTENTPROVIDER_OFFICIALNAME, Arrays.asList(new Param().setValue(cp.getOfficialname()).setName(CCONTENTPROVIDER_OFFICIALNAME))); + } + if (cp.getOpenaireId() != null){ + params.put( OPENAIRE_ID, Arrays.asList(new Param().setValue(cp.getOpenaireId()).setName(OPENAIRE_ID))); + } + if(cp.getSelectioncriteria() != null){ + params.put( CCONTENTPROVIDER_SELCRITERIA, Arrays.asList(new Param().setValue(cp.toJson()).setName(CCONTENTPROVIDER_SELCRITERIA))); + + } + + concept.setParams(params); + dts.getConcepts().add(concept); + + + } + + } + + public void updateOrganization(String communityId, CommunityOrganization organization) throws CommunityException { + + final Context context = getContextMap().get(communityId); + Category orgs = context.getCategories().get(communityId + ORGANIZATION_ID_SUFFIX); + if (orgs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())){ + orgs.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId())) { + + + if (organization.getName() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_NAME)){ + concept.getParams().replace(CORGANIZATION_NAME, Arrays.asList(new Param() + .setName(CORGANIZATION_NAME).setValue(organization.getName()))); + } + else{ + concept.getParams().put(CORGANIZATION_NAME, Arrays.asList(new Param() + .setName(CORGANIZATION_NAME).setValue(organization.getName()))); + } + + } + if (organization.getLogo_url() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_LOGOURL)){ + concept.getParams().replace(CORGANIZATION_LOGOURL, Arrays.asList(new Param() + .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())))); + } + else{ + concept.getParams().put(CORGANIZATION_LOGOURL, Arrays.asList(new Param() + .setName(CORGANIZATION_LOGOURL).setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())))); + } + + } + + + if (organization.getWebsite_url() != null) { + if(concept.getParams().keySet().contains(CORGANIZATION_WEBSITEURL)){ + concept.getParams().replace(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param() + .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())))); + } + else{ + concept.getParams().put(CORGANIZATION_WEBSITEURL, Arrays.asList(new Param() + .setName(CORGANIZATION_WEBSITEURL).setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())))); + } + + } + + } + + }); + } + else{ + + Concept concept = new Concept(); + concept.setId(communityId + ORGANIZATION_ID_SUFFIX + ID_SEPARATOR + organization.getId()); + concept.setClaim(false); + concept.setLabel(""); + + Map> params = new TreeMap<>(); + + if (organization.getName() != null) { + params.put( CORGANIZATION_NAME, Arrays.asList(new Param().setValue(organization.getName()).setName(CORGANIZATION_NAME))); + } + if(organization.getLogo_url()!= null){ + + params.put( CORGANIZATION_LOGOURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes())).setName(CORGANIZATION_LOGOURL))); + } + if (organization.getWebsite_url() != null){ + params.put( CORGANIZATION_WEBSITEURL, Arrays.asList(new Param().setValue(Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes())).setName(CORGANIZATION_WEBSITEURL))); + } + + + concept.setParams(params); + orgs.getConcepts().add(concept); + + + } + + } + + public void updateZenodoCommunity(String communityId, CommunityZenodoCommunity zc) throws CommunityException { + final Context context = getContextMap().get(communityId); + Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX); + if (zcs.getConcepts().stream().map(c -> c.getId()).collect(Collectors.toList()) + .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())){ + zcs.getConcepts().forEach(concept -> { + if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) { + + + if (zc.getZenodoid() != null) { + if(concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)){ + concept.getParams().replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() + .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid()))); + } + else{ + concept.getParams().put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() + .setName(CZENODOCOMMUNITY_ID).setValue(zc.getZenodoid()))); + } + + } + + } + + }); + } + else{ + + Concept concept = new Concept(); + concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId()); + concept.setClaim(false); + + + Map> params = new TreeMap<>(); + + if (zc.getZenodoid() != null) { + params.put( CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID))); + concept.setLabel(zc.getZenodoid()); + }else{ + concept.setLabel(""); + } + concept.setParams(params); + zcs.getConcepts().add(concept); + + } + + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java new file mode 100644 index 00000000..9ac100f9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityConstants.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.community; + +import java.util.Set; + +import com.google.common.collect.Sets; + +public class CommunityConstants { + + public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); + + // common + public final static String OPENAIRE_ID = "openaireId"; + public final static String PIPE_SEPARATOR = "||"; + public final static String ID_SEPARATOR = "::"; + public final static String CSV_DELIMITER = ","; + public final static String CLABEL = "label"; + + // id suffixes + public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; + public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; + public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; + public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; + + // community summary + public final static String CSUMMARY_DESCRIPTION = "description"; + public final static String CSUMMARY_LOGOURL = "logourl"; + public final static String CSUMMARY_STATUS = "status"; + public final static String CSUMMARY_NAME = "name"; + public final static String CSUMMARY_MANAGER = "manager"; + public final static String CSUMMARY_ZENODOC = "zenodoCommunity"; + + // community profile + public final static String CPROFILE_SUBJECT = "subject"; + public final static String CPROFILE_CREATIONDATE = "creationdate"; + + // community project + public final static String CPROJECT_FUNDER = "funder"; + public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; + public final static String CPROJECT_FULLNAME = "projectfullname"; + public final static String CPROJECT_ACRONYM = "acronym"; + + // community content provider + public final static String CCONTENTPROVIDER_NAME = "name"; + public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; + public final static String CCONTENTPROVIDER_ENABLED = "enabled"; + public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; + public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; + + //community zenodo community + public final static String CZENODOCOMMUNITY_ID = "zenodoid"; + + //community organization + public final static String CORGANIZATION_NAME = "name"; + public final static String CORGANIZATION_LOGOURL = "logourl"; + public final static String CORGANIZATION_WEBSITEURL = "websiteurl"; + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java new file mode 100644 index 00000000..334a3db1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityContentprovider.java @@ -0,0 +1,100 @@ +package eu.dnetlib.openaire.community; + +import javax.validation.constraints.NotNull; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.gson.Gson; + +import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityContentprovider { + + @ApiModelProperty(value = "OpenAIRE identifier for this content provider, if available", required = false) + private String openaireId; + + @NotNull + @ApiModelProperty(value = "the community identifier this content provider belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "identifies this content provider within the context it belongs to", required = true) + private String id; + + @ApiModelProperty(value = "content provider name", required = false) + private String name; + + @NotNull + @ApiModelProperty(value = "content provider official name", required = true) + private String officialname; + + // @NotNull + @ApiModelProperty(value = "content provider selection criteria", required = false) + private SelectionCriteria selectioncriteria; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(final String communityId) { + this.communityId = communityId; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public SelectionCriteria getSelectioncriteria() { + + return this.selectioncriteria; + } + + public void setSelectioncriteria(final SelectionCriteria selectioncriteria) { + this.selectioncriteria = selectioncriteria; + + } + + @Override + public String toString() { + return String.format("id %s, name %s, selection criteria %s", this.id, this.name, toJson()); + } + + public String toJson() { + if (selectioncriteria == null) { return ""; } + return new Gson().toJson(selectioncriteria); + } + + public String toXML() { + if (selectioncriteria == null) { return ""; } + return ""; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java new file mode 100644 index 00000000..5d315dd7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityDetails.java @@ -0,0 +1,51 @@ +package eu.dnetlib.openaire.community; + +import java.util.Date; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityDetails extends CommunitySummary { + + @ApiModelProperty("date of creation for this community") + private Date creationDate; + + @ApiModelProperty("date of the last update for this communityu") + private Date lastUpdateDate; + + @ApiModelProperty("list of subjects (keywords) that characterise this community") + private List subjects; + + public CommunityDetails() { + } + + public CommunityDetails(final CommunitySummary summary) { + super(summary); + } + + public Date getCreationDate() { + return creationDate; + } + + public void setCreationDate(final Date creationDate) { + this.creationDate = creationDate; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(final List subjects) { + this.subjects = subjects; + } + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public void setLastUpdateDate(Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java new file mode 100644 index 00000000..db2cc2bd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityException.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.community; + +import java.io.IOException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) +public class CommunityException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -4961233580574761346L; + + public CommunityException(final String message) { + super(message); + } + + public CommunityException(final IOException e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java new file mode 100644 index 00000000..271c1cbe --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java @@ -0,0 +1,231 @@ +package eu.dnetlib.openaire.community; + +import java.text.ParseException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.escape.Escaper; +import com.google.common.xml.XmlEscapers; +import eu.dnetlib.openaire.community.selectioncriteria.SelectionCriteria; +import eu.dnetlib.openaire.context.Concept; +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.Param; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import static eu.dnetlib.openaire.common.Utils.escape; +import static eu.dnetlib.openaire.community.CommunityConstants.*; + +public class CommunityMappingUtils { + + private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; + + private static final Log log = LogFactory.getLog(CommunityMappingUtils.class); + + public static CommunitySummary asCommunitySummary(final Context c) { + final CommunitySummary summary = new CommunitySummary(); + + summary.setId(c.getId()); + summary.setShortName(c.getLabel()); + summary.setLastUpdateDate(c.getLastUpdateDate()); + summary.setCreationDate(c.getCreationDate()); + summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); + summary.setType(c.getType()); + + final Map> params = c.getParams(); + if (params.containsKey(CSUMMARY_DESCRIPTION)) { + summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); + } + if (params.containsKey(CSUMMARY_LOGOURL)) { + summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); + } + if (params.containsKey(CSUMMARY_STATUS)) { + summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); + } + if (params.containsKey(CSUMMARY_NAME)) { + summary.setName(asCsv(params.get(CSUMMARY_NAME))); + } + if (params.containsKey(CSUMMARY_ZENODOC)) { + summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); + } + + return summary; + } + + public static CommunityDetails asCommunityProfile(final Context c) { + + final CommunityDetails p = new CommunityDetails(asCommunitySummary(c)); + p.setLastUpdateDate(c.getLastUpdateDate()); + final Map> params = c.getParams(); + if (params.containsKey(CPROFILE_SUBJECT)) { + p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_CREATIONDATE)){ + try { + p.setCreationDate(org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern)); + }catch(ParseException e) { + e.printStackTrace(); + } + } + + return p; + } + + public static CommunityProject asCommunityProject(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityProject project = new CommunityProject(); + project.setCommunityId(communityId); + project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + project.setOpenaireId(firstValue(p, OPENAIRE_ID)); + project.setFunder(firstValue(p, CPROJECT_FUNDER)); + project.setGrantId(firstValue(p, CPROJECT_NUMBER)); + project.setName(firstValue(p, CPROJECT_FULLNAME)); + project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); + + return project; + } + + public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityContentprovider d = new CommunityContentprovider(); + d.setCommunityId(communityId); + d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + d.setOpenaireId(firstValue(p, OPENAIRE_ID)); + d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); + d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); + d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); + return d; + } + + public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c){ + final CommunityZenodoCommunity z = new CommunityZenodoCommunity(); + final Map> p = c.getParams(); + z.setCommunityId(communityId); + z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + z.setZenodoid(firstValue(p,CZENODOCOMMUNITY_ID)); + //z.setName(c.getLabel()); + return z; + } + + + public static CommunityOrganization asCommunityOrganization(String id, Concept c) { + final Map> p = c.getParams(); + final CommunityOrganization o = new CommunityOrganization(); + o.setCommunityId(id); + o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + o.setName(firstValue(p,CORGANIZATION_NAME)); + o.setLogo_url(getDecodedUrl(firstValue(p,CORGANIZATION_LOGOURL))); + o.setWebsite_url(getDecodedUrl(firstValue(p,CORGANIZATION_WEBSITEURL))); + + return o; + } + + private static String getDecodedUrl(final String encoded_url){ + if(encoded_url == null){ + return encoded_url; + } + return new String(Base64.getDecoder().decode(encoded_url)); + } + + + private static List splitValues(final Stream stream, final String separator) { + return stream.map(s -> s.split(separator)) + .map(Arrays::asList) + .flatMap(List::stream) + .filter(StringUtils::isNotBlank) + .map(StringUtils::trim) + .collect(Collectors.toList()); + } + + private static String firstValue(final Map> p, final String paramName) { + return asValues(p.get(paramName)).findFirst().orElse(null); + } + + private static String asCsv(final List params) { + return asValues(params) + .collect(Collectors.joining(CSV_DELIMITER)); + } + + private static Stream asValues(final List params) { + return params == null ? Stream.empty() : params.stream() + .map(Param::getValue) + .map(StringUtils::trim) + .distinct(); + } + + public static String asProjectXML(final String contextId, final CommunityProject project) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), PROJECTS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(project.getId())), escape(esc, project.getAcronym()))); + sb.append(paramXML(CPROJECT_FULLNAME, project.getName())); + sb.append(paramXML(CPROJECT_ACRONYM, project.getAcronym())); + sb.append(paramXML(CPROJECT_NUMBER, project.getGrantId())); + sb.append(paramXML(CPROJECT_FUNDER, project.getFunder())); + sb.append(paramXML(OPENAIRE_ID, project.getOpenaireId())); + sb.append("\n"); + return sb.toString(); + } + + public static String asContentProviderXML(final String contextId, final CommunityContentprovider ccp) { + log.info("creating the XML for the content provider"); + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), CONTENTPROVIDERS_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(ccp.getId())), escape(esc, ccp.getName()))); + sb.append(paramXML(OPENAIRE_ID, ccp.getOpenaireId())); + sb.append(paramXML(CCONTENTPROVIDER_NAME, ccp.getName())); + sb.append(paramXML(CCONTENTPROVIDER_OFFICIALNAME, ccp.getOfficialname())); + sb.append(paramXML(CCONTENTPROVIDER_ENABLED,CCONTENTPROVIDERENABLED_DEFAULT)); + sb.append(paramXMLNoEscape(CCONTENTPROVIDER_SELCRITERIA, ccp.toXML())); + sb.append("\n"); + log.info(sb.toString()); + return sb.toString(); + } + + public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(zc.getId())), escape(esc, zc.getZenodoid()))); + + sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid())); + sb.append("\n"); + return sb.toString(); + } + + + public static String asOrganizationXML(final String contextId, CommunityOrganization organization) { + final Escaper esc = XmlEscapers.xmlAttributeEscaper(); + final StringBuilder sb = new StringBuilder(); + sb.append( + String.format( + "\n", + escape(esc, contextId), ORGANIZATION_ID_SUFFIX, ID_SEPARATOR, escape(esc, String.valueOf(organization.getId())), escape(esc, organization.getName()))); + sb.append(paramXML(CORGANIZATION_NAME, organization.getName())); + sb.append(paramXML(CORGANIZATION_LOGOURL, Base64.getEncoder().encodeToString(organization.getLogo_url().getBytes()))); + sb.append(paramXML(CORGANIZATION_WEBSITEURL,Base64.getEncoder().encodeToString(organization.getWebsite_url().getBytes()))); + sb.append("\n"); + return sb.toString(); + } + + + private static String paramXML(final String paramName, final String value) { + return String.format("%s\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value)); + } + + private static String paramXMLNoEscape(final String paramName, final String value) { + return String.format("%s\n", paramName, value); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java new file mode 100644 index 00000000..53eb016b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityNotFoundException.java @@ -0,0 +1,24 @@ +package eu.dnetlib.openaire.community; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.NOT_FOUND) +public class CommunityNotFoundException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -5605421323034135778L; + + public CommunityNotFoundException(final String msg) { + super(msg); + } + + public CommunityNotFoundException(final Exception e) { + super(e); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java new file mode 100644 index 00000000..3cf3403f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOpenAIRECommunities.java @@ -0,0 +1,42 @@ +package eu.dnetlib.openaire.community; + +import io.swagger.annotations.ApiModelProperty; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; + +public class CommunityOpenAIRECommunities { + + @NotNull + @ApiModelProperty(value = "the zenodo community identifier", required = true) + private String zenodoid; + + @NotNull + @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true) + private List openAirecommunitylist; + + public CommunityOpenAIRECommunities() { + this.zenodoid = ""; + openAirecommunitylist=new ArrayList<>(); + } + + public List getOpenAirecommunitylist() { + return openAirecommunitylist; + } + + public CommunityOpenAIRECommunities setOpenAirecommunitylist(List openAirecommunitylist) { + this.openAirecommunitylist = openAirecommunitylist; + return this; + } + + public String getZenodoid() { + return zenodoid; + } + + public CommunityOpenAIRECommunities setZenodoid(String zenodoid) { + this.zenodoid = zenodoid; + return this; + } + +} \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java new file mode 100644 index 00000000..7ced9784 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityOrganization.java @@ -0,0 +1,75 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModelProperty; +import javax.validation.constraints.NotNull; + +@JsonAutoDetect +public class CommunityOrganization { + @NotNull + @ApiModelProperty(value = "the community identifier this organization belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "name of the organization", required = true) + private String name; + + @NotNull + @ApiModelProperty(value = "identifies this organization within the context it belongs to", required = true) + private String id; + + @NotNull + @ApiModelProperty(value="url of the logo for this organization", required = true) + private String logo_url; + + + @NotNull + @ApiModelProperty(value="website url for this organization", required = true) + private String website_url; + + public String getCommunityId() { + return communityId; + } + + public CommunityOrganization setCommunityId(String communityId) { + this.communityId = communityId; + return this; + } + + public String getName() { + return name; + } + + public CommunityOrganization setName(String name) { + this.name = name; + return this; + } + + public String getId() { + return id; + } + + public CommunityOrganization setId(String id) { + this.id = id; + return this; + } + + public String getLogo_url() { + return logo_url; + } + + public CommunityOrganization setLogo_url(String logo_url) { + this.logo_url = logo_url; + return this; + } + + public String getWebsite_url() { + return website_url; + } + + public CommunityOrganization setWebsite_url(String website_url) { + this.website_url = website_url; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java new file mode 100644 index 00000000..205e844a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityProject.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityProject { + + @ApiModelProperty(value = "OpenAIRE identifier for this project, if available", required = false) + private String openaireId; + + @ApiModelProperty(value = "the community identifier this project belongs to", required = true) + private String communityId; + + @ApiModelProperty(value = "identifies this project within the context it belongs to", required = true) + private String id; + + @ApiModelProperty(value = "project name", required = true) + private String name; + + @ApiModelProperty(value = "project acronym", required = false) + private String acronym; + + @ApiModelProperty(value = "project funder", required = true) + private String funder; + + @ApiModelProperty(value = "project grant id", required = true) + private String grantId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(final String communityId) { + this.communityId = communityId; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getFunder() { + return funder; + } + + public void setFunder(final String funder) { + this.funder = funder; + } + + public String getGrantId() { + return grantId; + } + + public void setGrantId(final String grantId) { + this.grantId = grantId; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java new file mode 100644 index 00000000..e3b4902a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityStatus.java @@ -0,0 +1,17 @@ +package eu.dnetlib.openaire.community; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public enum CommunityStatus { + + @ApiModelProperty("restricted visibility") + hidden, + + @ApiModelProperty("visible only to RCD managers") + manager, + + @ApiModelProperty("visible to RCD managers and to the community users") + all +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java new file mode 100644 index 00000000..e504583e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunitySummary.java @@ -0,0 +1,174 @@ +package eu.dnetlib.openaire.community; + +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunitySummary { + + @ApiModelProperty("identifies the community") + protected String id; + + @ApiModelProperty("values for this field reflect the index field _community_ in the index, e.g. 'egi||EGI Federation'") + protected String queryId; + + @ApiModelProperty("community type") + protected String type; + + @ApiModelProperty("community name") + protected String name; + + @ApiModelProperty("community short name") + protected String shortName; + + @ApiModelProperty("community creation date") + protected Date creationDate; + + @ApiModelProperty("community last update date") + protected Date lastUpdateDate; + + @ApiModelProperty("community description") + protected String description; + + @ApiModelProperty("http url for the community logo") + protected String logoUrl; + + @ApiModelProperty("status of the community, drives its visibility") + protected CommunityStatus status; + + @ApiModelProperty("Zenodo community associated to this community") + protected String zenodoCommunity; + + public CommunitySummary() {} + + public CommunitySummary( + final String id, + final String queryId, + final String type, + final String name, + final String shortName, + final Date creationDate, + final Date lastUpdateDate, + final String description, + final String logoUrl, + final CommunityStatus status, + final String zenodoCommunity) { + this.id = id; + this.queryId = queryId; + this.type = type; + this.name = name; + this.shortName = shortName; + this.creationDate = creationDate; + this.lastUpdateDate = lastUpdateDate; + this.description = description; + this.logoUrl = logoUrl; + this.status = status; + this.zenodoCommunity = zenodoCommunity; + } + + public CommunitySummary(final CommunitySummary summary) { + this(summary.getId(), + summary.getQueryId(), + summary.getType(), + summary.getName(), + summary.getShortName(), + summary.getCreationDate(), + summary.getLastUpdateDate(), + summary.getDescription(), + summary.getLogoUrl(), + summary.getStatus(), + summary.getZenodoCommunity()); + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getQueryId() { + return queryId; + } + + public void setQueryId(final String queryId) { + this.queryId = queryId; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortName() { + return shortName; + } + + public void setShortName(final String shortName) { + this.shortName = shortName; + } + + public Date getCreationDate() { + return creationDate; + } + + public void setCreationDate(final Date creationDate) { + this.creationDate = creationDate; + } + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public void setLastUpdateDate(final Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getLogoUrl() { + return logoUrl; + } + + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + } + + public CommunityStatus getStatus() { + return status; + } + + public void setStatus(final CommunityStatus status) { + this.status = status; + } + + public String getZenodoCommunity() { + return zenodoCommunity; + } + + public void setZenodoCommunity(final String zenodoCommunity) { + this.zenodoCommunity = zenodoCommunity; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java new file mode 100644 index 00000000..08ca7a26 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityWritableProperties.java @@ -0,0 +1,96 @@ +package eu.dnetlib.openaire.community; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityWritableProperties { + + @ApiModelProperty("community name") + private String name; + + @ApiModelProperty("community short name") + private String shortName; + + @ApiModelProperty("community description") + private String description; + + @ApiModelProperty("http url for the community logo") + private String logoUrl; + + @ApiModelProperty("list of subjects (keywords) that characterise this community") + private List subjects; + + @ApiModelProperty("status of the community, drives its visibility") + private CommunityStatus status; + + @ApiModelProperty("id of the main Zenodo community") + private String mainZenodoCommunity; + + + public static CommunityWritableProperties fromDetails(final CommunityDetails details) { + CommunityWritableProperties p = new CommunityWritableProperties(); + p.setName(details.getName()); + p.setShortName(details.getShortName()); + p.setDescription(details.getDescription()); + p.setLogoUrl(details.getLogoUrl()); + p.setSubjects(details.getSubjects()); + p.setStatus(details.getStatus()); + p.setMainZenodoCommunity(details.getZenodoCommunity()); + return p; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortName() { + return shortName; + } + + public void setShortName(final String shortName) { + this.shortName = shortName; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getLogoUrl() { + return logoUrl; + } + + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(final List subjects) { + this.subjects = subjects; + } + + public CommunityStatus getStatus() { + return status; + } + + public void setStatus(final CommunityStatus status) { + this.status = status; + } + + public String getMainZenodoCommunity() { return mainZenodoCommunity; } + + public void setMainZenodoCommunity(String mainZenodoCommunity) { this.mainZenodoCommunity = mainZenodoCommunity; } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java new file mode 100644 index 00000000..c84c1692 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityZenodoCommunity.java @@ -0,0 +1,47 @@ +package eu.dnetlib.openaire.community; + +import javax.validation.constraints.NotNull; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +public class CommunityZenodoCommunity { + + @NotNull + @ApiModelProperty(value = "the community identifier this zenodo Community belongs to", required = true) + private String communityId; + + @NotNull + @ApiModelProperty(value = "Zenodo identifier for this community", required = true) + private String zenodoid; + + @NotNull + @ApiModelProperty(value = "identifies this zenodo community within the context it belongs to", required = true) + private String id; + + public String getZenodoid() { + return zenodoid; + } + + public void setZenodoid(String zenodoid) { + this.zenodoid = zenodoid; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCommunityId() { + return communityId; + } + + public void setCommunityId(String communityId) { + this.communityId = communityId; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java new file mode 100644 index 00000000..31c71535 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraint.java @@ -0,0 +1,45 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class Constraint implements Serializable { + + /** + * + */ + private static final long serialVersionUID = -5996232267609464747L; + + private String verb; + private String field; + private String value; + + public Constraint() {} + + public String getVerb() { + return verb; + } + + public void setVerb(final String verb) { + this.verb = verb; + } + + public String getField() { + return field; + } + + public void setField(final String field) { + this.field = field; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java new file mode 100644 index 00000000..a899095f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/Constraints.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class Constraints implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 2694950017620361195L; + + private List constraint; + + public Constraints() {} + + public List getConstraint() { + return constraint; + } + + public void setConstraint(final List constraint) { + this.constraint = constraint; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java new file mode 100644 index 00000000..0214e41a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteria.java @@ -0,0 +1,33 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.gson.Gson; + +@JsonAutoDetect +public class SelectionCriteria implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 4303936216579280542L; + + private List criteria; + + public SelectionCriteria() {} + + public List getCriteria() { + return criteria; + } + + public void setCriteria(final List criteria) { + this.criteria = criteria; + } + + public static SelectionCriteria fromJson(final String json) { + return new Gson().fromJson(json, SelectionCriteria.class); + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java new file mode 100644 index 00000000..1c029777 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Category.java @@ -0,0 +1,67 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Map; + +public class Category { + + private String id; + + private String label; + + private boolean claim; + + private Map> params; + + private List concepts; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isClaim() { + return claim; + } + + public boolean hasConcepts() { + return getConcepts() != null && !getConcepts().isEmpty(); + } + + public Map> getParams() { + return params; + } + + public List getConcepts() { + return concepts; + } + + public Category setId(final String id) { + this.id = id; + return this; + } + + public Category setLabel(final String label) { + this.label = label; + return this; + } + + public Category setClaim(final boolean claim) { + this.claim = claim; + return this; + } + + public Category setParams(final Map> params) { + this.params = params; + return this; + } + + public Category setConcepts(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java new file mode 100644 index 00000000..c2acb200 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/CategorySummary.java @@ -0,0 +1,38 @@ +package eu.dnetlib.openaire.context; + +public class CategorySummary { + + private String id; + + private String label; + + private boolean hasConcept; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isHasConcept() { + return hasConcept; + } + + public CategorySummary setId(final String id) { + this.id = id; + return this; + } + + public CategorySummary setLabel(final String label) { + this.label = label; + return this; + } + + public CategorySummary setHasConcept(final boolean hasConcept) { + this.hasConcept = hasConcept; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java new file mode 100644 index 00000000..ddbcdec1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Concept.java @@ -0,0 +1,67 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Map; + +public class Concept { + + private String id; + + private String label; + + private boolean claim; + + private Map> params; + + private List concepts; + + public boolean hasSubConcepts() { + return getConcepts() != null && !getConcepts().isEmpty(); + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public boolean isClaim() { + return claim; + } + + public Map> getParams() { + return params; + } + + public List getConcepts() { + return concepts; + } + + public Concept setId(final String id) { + this.id = id; + return this; + } + + public Concept setLabel(final String label) { + this.label = label; + return this; + } + + public Concept setClaim(final boolean claim) { + this.claim = claim; + return this; + } + + public Concept setParams(final Map> params) { + this.params = params; + return this; + } + + public Concept setConcepts(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java new file mode 100644 index 00000000..58600836 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ConceptSummary.java @@ -0,0 +1,51 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; + +public class ConceptSummary { + + private String id; + + private String label; + + public boolean hasSubConcept; + + private List concepts; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public List getConcepts() { + return concepts; + } + + public ConceptSummary setId(final String id) { + this.id = id; + return this; + } + + public ConceptSummary setLabel(final String label) { + this.label = label; + return this; + } + + public boolean isHasSubConcept() { + return hasSubConcept; + } + + public ConceptSummary setHasSubConcept(final boolean hasSubConcept) { + this.hasSubConcept = hasSubConcept; + return this; + } + + public ConceptSummary setConcept(final List concepts) { + this.concepts = concepts; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java new file mode 100644 index 00000000..badc1ff1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Context.java @@ -0,0 +1,91 @@ +package eu.dnetlib.openaire.context; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class Context { + + private String id; + + private String label; + + private String type; + + private Date creationDate; + + private Date lastUpdateDate; + + private Map> params; + + private Map categories; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getType() { + return type; + } + + public Date getCreationDate() { + return creationDate; + } + + public Map> getParams() { + return params; + } + + public Map getCategories() { + return categories; + } + + + + public Context setId(final String id) { + this.id = id; + return this; + } + + public Context setLabel(final String label) { + this.label = label; + return this; + } + + public Context setType(final String type) { + this.type = type; + return this; + } + + public Context setCreationDate(final Date dateofcreation) { + this.creationDate = dateofcreation; + return this; + } + + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public Context setLastUpdateDate(Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + return this; + } + + + public Context setParams(final Map> params) { + this.params = params; + return this; + } + + public Context setCategories(final Map categories) { + this.categories = categories; + return this; + } + +} + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java new file mode 100644 index 00000000..64aae1fe --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiController.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.context; + +import java.util.List; +import java.util.Optional; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Context API", description = "the OpenAIRE Context API") +public class ContextApiController { + + @Autowired + private ContextApiCore contextApiCore; + + @RequestMapping(value = "/contexts", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list brief information about all the context profiles", + notes = "list brief information about all the context profiles.", + tags = { }, + response = ContextSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ContextSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listContexts(@RequestParam(required = false, defaultValue = "") List type) throws ContextException { + return contextApiCore.listContexts(type); + } + + @RequestMapping(value = "/context/{contextId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the categories defined within a context", + notes = "list the categories defined within a context", + tags = { }, + response = CategorySummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = CategorySummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listCategories( + @PathVariable final String contextId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listCategories(contextId, allFilter); + } + + @RequestMapping(value = "/context/category/{categoryId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the concepts defined within a category", + notes = "list the concepts defined within a category", + tags = { }, + response = ConceptSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listConcepts( + @PathVariable final String categoryId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listConcepts(categoryId, allFilter); + } + + @RequestMapping(value = "/context/category/concept/{conceptId}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "list the concepts defined within a category", + notes = "list the concepts defined within a category", + tags = { }, + response = ConceptSummary[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ConceptSummary[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ContextException.class) }) + public List listSubConcepts( + @PathVariable final String conceptId, + @RequestParam(required = false, defaultValue = "false") final Boolean all) throws ContextException { + + Boolean allFilter = Optional.ofNullable(all).orElse(false); + return contextApiCore.listSubConcepts(conceptId, allFilter); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java new file mode 100644 index 00000000..9cd2e859 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java @@ -0,0 +1,113 @@ +package eu.dnetlib.openaire.context; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.common.ISClient; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.context", havingValue = "true") +public class ContextApiCore { + + private static final String SEPARATOR = "::"; + + @Autowired + private ISClient isClient; + + public List listContexts(final List type) throws ContextException { + return getContextMap(type).values() + .stream() + .map(c -> new ContextSummary() + .setId(c.getId()) + .setType(c.getType()) + .setLabel(c.getLabel()) + .setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : "")) + .collect(Collectors.toList()); + } + + public List listCategories(final String contextId, final Boolean all) throws ContextException { + final Stream categories = getContextMap().get(contextId).getCategories().values().stream(); + return all ? asCategorySummaries(categories) : asCategorySummaries(categories.filter(Category::isClaim)); + } + + private List asCategorySummaries(final Stream categories) { + return categories + .map(c -> new CategorySummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasConcept(c.hasConcepts())) + .collect(Collectors.toList()); + } + + public List listConcepts(final String categoryId, final Boolean all) throws ContextException { + final String contextId = StringUtils.substringBefore(categoryId, SEPARATOR); + final Stream concepts = getContextMap().get(contextId) + .getCategories() + .get(categoryId) + .getConcepts() + .stream(); + + return all ? asConceptSummaries(concepts) : asConceptSummaries(concepts.filter(Concept::isClaim)); + } + + private List asConceptSummaries(final Stream concepts) { + return concepts + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts())) + .collect(Collectors.toList()); + } + + public List listSubConcepts(final String conceptId, final Boolean all) throws ContextException { + final List ids = Splitter.on(SEPARATOR).splitToList(conceptId); + if (ids.size() < 3) { throw new ContextException(""); } + + final String contextId = ids.get(0); + final String categoryId = contextId + SEPARATOR + ids.get(1); + + final Stream concepts = getContextMap().get(contextId) + .getCategories() + .get(categoryId) + .getConcepts() + .stream() + .filter(c -> conceptId.equals(c.getId())); + + return all ? mapConcepts(concepts.filter(Concept::isClaim).collect(Collectors.toList())) : mapConcepts(concepts.collect(Collectors.toList())); + } + + private List mapConcepts(final List concepts) { + if (concepts == null || concepts.isEmpty()) { return null; } + return concepts.stream() + .map(c -> new ConceptSummary() + .setId(c.getId()) + .setLabel(c.getLabel()) + .setHasSubConcept(c.hasSubConcepts()) + .setConcept(mapConcepts(c.getConcepts()))) + .collect(Collectors.toList()); + } + + private Map getContextMap() throws ContextException { + return getContextMap(Lists.newArrayList()); + } + + private Map getContextMap(final List type) throws ContextException { + try { + return isClient.getContextMap(type); + } catch (final IOException e) { + throw new ContextException(e); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java new file mode 100644 index 00000000..7d092b96 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextException.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.context; + +import java.io.IOException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) +public class ContextException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -5489369676370127052L; + + public ContextException(final String message) { + super(message); + } + + public ContextException(final IOException e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java new file mode 100644 index 00000000..c3537343 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java @@ -0,0 +1,113 @@ +package eu.dnetlib.openaire.context; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.dom4j.Element; +import org.dom4j.Node; + +import com.google.common.base.Functions; +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.funders.domain.FunderDetails; + +public class ContextMappingUtils { + + private static final List DATE_PATTERN = Lists.newArrayList("yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd'T'hh:mm:ssXXX", "yyyy-MM-dd'T'hh:mm:ss+00:00"); + + public static Context parseContext(final String s, final Queue errors) { + try { + final Document doc = DocumentHelper.parseText(s); + final Element eContext = (Element) doc.selectSingleNode("/RESOURCE_PROFILE/BODY/CONFIGURATION/context"); + final String creationDate = eContext.valueOf("./param[./@name='creationdate']/text()"); + final Context c = new Context() + .setId(eContext.attributeValue("id")) + .setLabel(eContext.attributeValue("label")) + .setType(eContext.attributeValue("type")) + .setLastUpdateDate(asDate(doc.valueOf("/RESOURCE_PROFILE/HEADER/DATE_OF_CREATION/@value"))) + .setParams(parseParams(eContext)) + .setCategories(parseCategories(eContext)); + // the creation date will be added in the param elements of the community profile. Funders may not have it, hence the check. + if (StringUtils.isNotBlank(creationDate)) { + c.setCreationDate(asDate(creationDate)); + } + return c; + } catch (final DocumentException e) { + errors.add(e); + return new Context(); + } + } + + private static Date asDate(final String s) { + for (final String pattern : DATE_PATTERN) { + try { + return DateUtils.parseDate(s, pattern); + } catch (final ParseException e) {} + } + return null; + } + + private static Map parseCategories(final Element eContext) { + final List eCategory = eContext.selectNodes("//category"); + return eCategory.stream() + .map(n -> (Element) n) + .map(eCat -> new Category() + .setClaim(getClaim(eCat)) + .setId(eCat.attributeValue("id")) + .setLabel(eCat.attributeValue("label")) + .setParams(parseParams(eCat)) + .setConcepts(parseConcepts(eCat))) + .collect(Collectors.toMap(Category::getId, Functions.identity())); + } + + private static List parseConcepts(final Element eCategory) { + final List eConcepts = eCategory.selectNodes("./concept"); + return eConcepts.stream() + .map(n -> (Element) n) + .map(eCon -> new Concept() + .setClaim(getClaim(eCon)) + .setId(eCon.attributeValue("id")) + .setLabel(eCon.attributeValue("label")) + .setParams(parseParams(eCon)) + .setConcepts(parseConcepts(eCon))) + .collect(Collectors.toList()); + } + + private static Boolean getClaim(final Element eCon) { + final String claim = eCon.attributeValue("claim"); + return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false"); + } + + private static Map> parseParams(final Element e) { + final List params = e.selectNodes("./param"); + return params.stream() + .map(n -> (Element) n) + .map(p -> new Param() + .setName(p.attributeValue("name")) + .setValue(p.getTextTrim())) + .collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> { + final List p = new ArrayList<>(p1); + p.addAll(p2); + return p; + })); + } + + public static FunderDetails asFunderDetails(final Context c) { + return new FunderDetails() + .setId(c.getId()) + .setName(c.getLabel()) + .setShortname(c.getId()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java new file mode 100644 index 00000000..ba87774c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextNotFoundException.java @@ -0,0 +1,23 @@ +package eu.dnetlib.openaire.context; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseBody +@ResponseStatus(value = HttpStatus.NOT_FOUND) +public class ContextNotFoundException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -2026506752817353752L; + + public ContextNotFoundException(final String msg) { + super(msg); + } + + public ContextNotFoundException(final Exception e) { + super(e); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java new file mode 100644 index 00000000..9e974a3d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextSummary.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.context; + +public class ContextSummary { + + private String id; + + private String label; + + private String type; + + private String status; + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getType() { + return type; + } + + public String getStatus() { + return status; + } + + public ContextSummary setId(final String id) { + this.id = id; + return this; + } + + public ContextSummary setLabel(final String label) { + this.label = label; + return this; + } + + public ContextSummary setType(final String type) { + this.type = type; + return this; + } + + public ContextSummary setStatus(final String status) { + this.status = status; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java new file mode 100644 index 00000000..e83a6fd2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/Param.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.context; + +public class Param { + + private String name; + + private String value; + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + public Param setName(final String name) { + this.name = name; + return this; + } + + public Param setValue(final String value) { + this.value = value; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java new file mode 100755 index 00000000..460a9970 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -0,0 +1,407 @@ +package eu.dnetlib.openaire.dsm; + +import static eu.dnetlib.openaire.common.ExporterConstants.API; +import static eu.dnetlib.openaire.common.ExporterConstants.DS; +import static eu.dnetlib.openaire.common.ExporterConstants.M; +import static eu.dnetlib.openaire.common.ExporterConstants.R; +import static eu.dnetlib.openaire.common.ExporterConstants.W; + +import java.util.List; + +import javax.validation.Valid; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.OperationManager; +import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.Response; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; +import eu.dnetlib.openaire.vocabularies.Country; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; + +@RestController +@CrossOrigin(origins = { + "*" +}) +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API") +public class DsmApiController extends AbstractExporterController { + + @Autowired + private DsmCore dsmCore; + + @RequestMapping(value = "/ds/countries", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "list the datasource countries", notes = "list the datasource countries", tags = { + DS, R + }, response = Country[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = Country[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public List listCountries() throws DsmException { + return dsmCore.listCountries(); + } + + @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { + DS, R + }, response = DatasourceDetailResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceDetailResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public DatasourceDetailResponse searchDsDetails( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource details.", tags = { + DS, R + }, response = AggregationHistoryResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = AggregationHistoryResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId); + return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp); + } + + @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search datasources", notes = "Returns list of Datasource basic info.", tags = { + DS, R + }, response = DatasourceSnippetResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public DatasourceSnippetResponse searchSnippet( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search among registered datasources", notes = "Returns list of Datasource basic info.", tags = { + DS, + R + }, response = DatasourceSnippetResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceSnippetResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public DatasourceSnippetResponse searchRegistered( + @RequestParam final RequestSort requestSortBy, + @RequestParam final RequestSortOrder order, + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + final StopWatch stop = StopWatch.createStarted(); + final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size); + return prepareResponse(page, size, stop, rsp); + } + + @RequestMapping(value = "/ds/recentregistered/{size}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "return the latest datasources that were registered through Provide", notes = "Returns list of Datasource basic info.", tags = { + DS, + R + }, response = SimpleResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = DatasourceResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); + return prepareResponse(1, size, stop, rsp); + } + + @RequestMapping(value = "/ds/countregistered", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "return the number of datasources registered after the given date", notes = "Returns a number.", tags = { + DS, + R + }, response = Long.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = Long.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public Long countRegistered(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + return dsmCore.countRegisteredAfter(fromDate, typologyFilter); + } + + @RequestMapping(value = "/ds/api/{dsId}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get the list of API for a given datasource", notes = "Returns the list of API for a given datasource.", tags = { + API, + R + }, response = ApiDetailsResponse.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ApiDetailsResponse.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public ApiDetailsResponse getApi( + @PathVariable final String dsId) throws DsmException { + + final StopWatch stop = StopWatch.createStarted(); + final ApiDetailsResponse rsp = dsmCore.getApis(dsId); + return prepareResponse(0, rsp.getApi().size(), stop, rsp); + } + + @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { + "application/json" + }, method = RequestMethod.POST) + @ApiOperation(value = "search for the list of base URLs of Datasource APIs managed by a user", notes = "Returns the list of base URLs of Datasource APIs managed by a user", tags = { + DS, API, R + }, response = String[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public List searchBaseUrls( + @RequestBody final RequestFilter requestFilter, + @PathVariable final int page, + @PathVariable final int size) throws DsmException { + + return dsmCore.findBaseURLs(requestFilter, page, size); + } + + @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE) + @ApiOperation(value = "delete an API", notes = "delete an API, if removable", tags = { + API, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Api not found", response = ErrorMessage.class), + @ApiResponse(code = 403, message = "Api not removable", response = ErrorMessage.class), + @ApiResponse(code = 500, message = "DSM Server error", response = ErrorMessage.class) + }) + public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException { + dsmCore.deleteApi(apiId); + } + + @RequestMapping(value = "/ds/manage", method = RequestMethod.POST) + @ApiOperation(value = "set the managed status for a given datasource", notes = "set the managed status for a given datasource", tags = { + DS, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void setManaged( + @RequestParam final String id, + @RequestParam final boolean managed) throws DsmException { + + dsmCore.setManaged(id, managed); + } + + @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET) + @ApiOperation(value = "get the datasource managed status", notes = "get the datasource managed status", tags = { + DS, R + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public boolean isManaged(@PathVariable final String id) throws DsmException { + return dsmCore.isManaged(id); + } + + @RequestMapping(value = "/ds/add", method = RequestMethod.POST) + @ApiOperation(value = "add a new Datasource", notes = "add a new Datasource", tags = { + DS, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 400, message = "Malformed request", response = ErrorMessage[].class), + @ApiResponse(code = 500, message = "Unexpected error", response = ErrorMessage.class) + }) + public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException { + + if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API + throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", datasource.getId())); + } + dsmCore.save(datasource); + } + + @RequestMapping(value = "/ds/update", method = RequestMethod.POST) + @ApiOperation(value = "update Datasource details", notes = "update Datasource details", tags = { + DS, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void updateDatasource( + @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException { + + dsmCore.updateDatasource(ds); + } + + @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST) + @ApiOperation(value = "update the base URL of a datasource interface", notes = "update the base URL of a datasource interface", tags = { + API, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void updateBaseUrl( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String baseUrl) throws DsmException { + + dsmCore.updateApiBaseurl(dsId, apiId, baseUrl); + } + + @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST) + @ApiOperation(value = "update the compatibility of a datasource interface", notes = "update the compatibility of a datasource interface", tags = { + API, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void updateCompliance( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String compliance, + @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException { + + dsmCore.updateApiCompatibility(dsId, apiId, compliance, override); + } + + @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST) + @ApiOperation(value = "update the OAI set of a datasource interface", notes = "update the OAI set of a datasource interface", tags = { + API, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void updateOaiSetl( + @RequestParam final String dsId, + @RequestParam final String apiId, + @RequestParam final String oaiSet) throws DsmException { + + dsmCore.updateApiOaiSet(dsId, apiId, oaiSet); + } + + @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST) + @ApiOperation(value = "adds a new Interface to one Datasource", notes = "adds an Interface to one Datasource", tags = { + API, W + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void addApi(@RequestBody final ApiDetails api) throws DsmException { + if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); } + dsmCore.addApi(api); + } + + // MANAGEMENT + + @Autowired + private OperationManager operationManager; + + @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET) + @ApiOperation(value = "get the number of pending operations", notes = "get the number of pending operations", tags = { + R, M + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public int getOps() throws DsmException { + return operationManager.getOpSize(); + } + + @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST) + @ApiOperation(value = "interrupts the pending operations", notes = "return the number of interrupted operations", tags = { + W, M + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public int killOps() throws DsmException { + return operationManager.dropAll(); + } + + @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST) + @ApiOperation(value = "drop the caches", notes = "drop the internal caches", tags = { + W, M + }) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK"), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public void dropCache() throws DsmException { + dsmCore.dropCaches(); + } + + // HELPERS + private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { + rsp.getHeader() + .setTime(stopWatch.getTime()) + .setPage(page) + .setSize(size); + return rsp; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java new file mode 100644 index 00000000..0d7e9107 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -0,0 +1,296 @@ +package eu.dnetlib.openaire.dsm; + +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDbEntry; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.asDetails; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.copyNonNullProperties; +import static eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils.createId; + +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.Page; +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.common.ISClient; +import eu.dnetlib.openaire.community.CommunityClient; +import eu.dnetlib.openaire.dsm.dao.DatasourceDao; +import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; +import eu.dnetlib.openaire.dsm.dao.ResponseUtils; +import eu.dnetlib.openaire.dsm.dao.VocabularyClient; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DsmCore { + + private static final Log log = LogFactory.getLog(DsmCore.class); + + @Autowired + private MongoLoggerClient mongoLoggerClient; + + @Autowired + private ISClient isClient; + + @Autowired + private VocabularyClient vocabularyClient; + + @Autowired + private DatasourceDao dsDao; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Autowired + private CommunityClient communityClient; + + public List listCountries() throws DsmException { + try { + return dsDao.listCountries(); + } catch (final Throwable e) { + log.error("error listing countries", e); + throw e; + } + } + + public DatasourceDetailResponse searchDsDetails(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + + try { + final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.detailsResponse(dsPage.map(d -> asDetails(d)).getContent(), dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + try { + final Page dsPage = dsDao.search(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public DatasourceSnippetResponse searchRegistered(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + try { + final Page dsPage = dsDao.searchRegistered(requestSortBy, order, requestFilter, page, size); + return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements()); + } catch (final Throwable e) { + log.error("error searching datasources", e); + throw e; + } + } + + public List findBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException { + try { + return dsDao.findApiBaseURLs(requestFilter, page, size); + } catch (final Throwable e) { + log.error("error searching datasource base urls", e); + throw e; + } + } + + public ApiDetailsResponse getApis(final String dsId) throws DsmException { + try { + final String eoscType = dsDao.getDs(dsId).getEoscDatasourceType(); + final List apis = dsDao.getApis(dsId); + final List api = apis.stream() + .map(DsmMappingUtils::asDetails) + .map(a -> a.setEoscDatasourceType(eoscType)) + .collect(Collectors.toList()); + return ResponseUtils.apiResponse(api, api.size()); + } catch (final Throwable e) { + log.error(String.format("error searching datasource api %s", dsId), e); + throw e; + } + } + + public void setManaged(final String dsId, final boolean managed) throws DsmException { + log.info(String.format("updated ds '%s' managed with '%s'", dsId, managed)); + dsDao.setManaged(dsId, managed); + } + + public boolean isManaged(final String dsId) throws DsmException { + return dsDao.isManaged(dsId); + } + + public boolean exist(final DatasourceDetails d) throws DsmException { + return dsDao.existDs(d.getId()); + } + + public void save(final DatasourceDetails d) throws DsmException { + try { + dsDao.saveDs(asDbEntry(d)); + } catch (final Throwable e) { + log.error(ExceptionUtils.getStackTrace(e)); + throw e; + } + } + + public void updateDatasource(final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException { + try { + // initialize with current values from DB + final DatasourceDbEntry ds = dsDao.getDs(d.getId()); + + if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); } + + final DatasourceDbEntry update = asDbEntry(d); + if (d.getIdentities() != null) { + final Set identities = new HashSet<>( + Stream.of(update.getIdentities(), ds.getIdentities()) + .flatMap(Collection::stream) + .collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1)) + .values()); + copyNonNullProperties(update, ds); + ds.setIdentities(identities); + } else { + copyNonNullProperties(update, ds); + } + + dsDao.saveDs(ds); + } catch (final Throwable e) { + log.error(ExceptionUtils.getStackTrace(e)); + throw e; + } + } + + // TODO remove if unused + public void deleteDs(final String dsId) throws DsmException { + log.info(String.format("deleted datasource '%s'", dsId)); + dsDao.deleteDs(dsId); + } + + // API + + public void updateApiOaiSet(final String dsId, final String apiId, final String oaiSet) throws DsmException { + dsDao.upsertApiOaiSet(apiId, oaiSet); + } + + public void updateApiBaseurl(final String dsId, final String apiId, final String baseUrl) throws DsmException { + log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl)); + dsDao.updateApiBaseUrl(apiId, baseUrl); + } + + public void updateApiCompatibility(final String dsId, final String apiId, final String compliance, final boolean override) throws DsmException { + log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance)); + dsDao.updateCompliance(null, apiId, compliance, override); + } + + public void addApi(final ApiDetails api) throws DsmException { + if (StringUtils.isBlank(api.getId())) { + api.setId(createId(api)); + log.info(String.format("missing api id, created '%s'", api.getId())); + } + dsDao.addApi(asDbEntry(api)); + } + + public void deleteApi(final String apiId) throws DsmForbiddenException, DsmNotFoundException { + // TODO handle the api removal in case of associated workflows. + dsDao.deleteApi(null, apiId); + } + + public void dropCaches() { + mongoLoggerClient.dropCache(); + isClient.dropCache(); + vocabularyClient.dropCache(); + communityClient.dropCache(); + } + + // HELPERS ////////////// + + public SimpleResponse searchRecentRegistered(final int size) throws Throwable { + try { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); + + final List list = jdbcTemplate.query(sql, BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size); + + return ResponseUtils.simpleResponse(list); + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + + public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typeFilter)) { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%"); + } else { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, fromDate); + } + + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + + public AggregationHistoryResponse aggregationhistory(final String dsId) throws DsmException { + final List history = mongoLoggerClient.getAggregationHistory(dsId); + final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history); + rsp.setHeader(ResponseUtils.header(history.size())); + return rsp; + + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java new file mode 100644 index 00000000..044ed440 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ApiDbEntryRepository.java @@ -0,0 +1,53 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; +import javax.transaction.Transactional; + +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 15/06/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface ApiDbEntryRepository extends JpaRepository { + + @Query("select a from #{#entityName} a where a.datasource = ?1") + List findByDatasource(String dsId); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.baseurl = ?2 where a.id = ?1") + void setBaseurl(String id, String baseurl); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.compatibility = ?2 where a.id = ?1") + void updateCompatibility(String apiId, String compatibility); + + @Modifying + @Transactional + @Query("update #{#entityName} a set a.compatibilityOverride = ?2 where a.id = ?1") + void updateCompatibilityOverride(String apiId, String compatibility); + + @Modifying + @Transactional + @Query(value = "update dsm_apiparams ap set value = ?2 where ap.param = 'set' and ap.api = ?1", nativeQuery = true) + void updateOaiSet(String apiId, String oaiSet); + + @Modifying + @Transactional + @Query(value = "insert into dsm_apiparams(api, param, value, _dnet_resource_identifier_) values(?1, ?2, ?3, ?1||'@@'||?2)", nativeQuery = true) + void addApiParam(String apiId, String param, String value); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.removable = ?2 where d.datasource = ?1") + void setRemovable(String id, boolean removable); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java new file mode 100644 index 00000000..221810f1 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/CountryTermRepository.java @@ -0,0 +1,15 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.openaire.dsm.domain.db.CountryTerm; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 19/04/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface CountryTermRepository extends JpaRepository { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java new file mode 100644 index 00000000..af13e141 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceApiDbEntryRepository.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.stereotype.Repository; + +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface DatasourceApiDbEntryRepository extends JpaRepository, JpaSpecificationExecutor { + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java new file mode 100644 index 00000000..5f40524d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDao.java @@ -0,0 +1,60 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; + +import org.springframework.data.domain.Page; + +import eu.dnetlib.enabling.datasources.common.Api; +import eu.dnetlib.enabling.datasources.common.Datasource; +import eu.dnetlib.enabling.datasources.common.DatasourceManagerCommon; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.vocabularies.Country; + +public interface DatasourceDao, API extends Api> extends DatasourceManagerCommon { + + // DATASOURCE + + List listCountries() throws DsmException; + + boolean existDs(final String dsId) throws DsmException; + + Page search(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException; + + Page searchRegistered(RequestSort requestSortBy, RequestSortOrder order, RequestFilter requestFilter, int page, int size) throws DsmException; + + void updateName(String dsId, String officialname, String englishname) throws DsmException; + + void updateLogoUrl(String dsId, String logourl) throws DsmException; + + void updateCoordinates(String dsId, Double latitude, Double longitude) throws DsmException; + + void updateTimezone(String dsId, String timezone) throws DsmException; + + void updateEoscDatasourceType(String dsId, String timezone) throws DsmException; + + void updateRegisteringUser(String dsId, String registeredBy) throws DsmException; + + void updatePlatform(String dsId, String platform) throws DsmException; + + // API + + List findApiBaseURLs(RequestFilter requestFilter, int page, int size) throws DsmException; + + /** + * Insert the oai set in case it does not exists, updates it otherwise + * + * @param apiId + * @param oaiSet + * @return true in case of insert, false in case of update + * @throws DsmException + */ + boolean upsertApiOaiSet(String apiId, String oaiSet) throws DsmException; + + void updateApiBaseUrl(String apiId, String baseUrl) throws DsmException; + + @Override + void addApi(final API api) throws DsmException; +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java new file mode 100644 index 00000000..0f9cf5aa --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDaoImpl.java @@ -0,0 +1,274 @@ +package eu.dnetlib.openaire.dsm.dao; + +import static eu.dnetlib.openaire.common.ExporterConstants.OAI; +import static eu.dnetlib.openaire.common.ExporterConstants.SET; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.apiSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsRegisteredbyNotNullSpec; +import static eu.dnetlib.openaire.dsm.dao.DatasourceSpecs.dsSpec; + +import java.sql.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.persistence.EntityNotFoundException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import com.google.common.collect.Lists; + +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.ApiParamDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; +import eu.dnetlib.openaire.vocabularies.Vocabulary; + +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DatasourceDaoImpl implements DatasourceDao { + + private static final Log log = LogFactory.getLog(DatasourceDao.class); + + @Autowired + private DnetOpenaireExporterProperties config; + + @Autowired + private CountryTermRepository countryTermRepository; + + @Autowired + private DatasourceDbEntryRepository dsRepository; + + @Autowired + private ApiDbEntryRepository apiRepository; + + @Autowired + private DatasourceApiDbEntryRepository dsApiRepository; + + @Autowired + private VocabularyClient vocabularyClient; + + @Override + public List listCountries() throws DsmException { + final List countries = Lists.newArrayList(); + final Vocabulary v = vocabularyClient.getCountries(); + countries.addAll(countryTermRepository.findAll() + .stream() + .filter(Objects::nonNull) + .map(t -> new Country(t.getTerm(), v.getEnglishName(t.getTerm()))) + .collect(Collectors.toList())); + return countries; + } + + @Override + public Page search(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + + final Specification spec = dsSpec(requestSortBy, order, requestFilter); + return dsRepository.findAll(spec, PageRequest.of(page, size)); + } + + @Override + public Page searchRegistered(final RequestSort requestSortBy, + final RequestSortOrder order, + final RequestFilter requestFilter, + final int page, + final int size) + throws DsmException { + + final Specification spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec()); + return dsRepository.findAll(spec, PageRequest.of(page, size)); + } + + @Override + public DatasourceDbEntry getDs(final String dsId) throws DsmException { + return dsRepository.findById(dsId).orElseThrow(() -> new DsmException("Datasource not found. ID: " + dsId)); + } + + @Override + public DatasourceDbEntry getDsByNsPrefix(final String prefix) throws DsmException { + return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix)); + } + + @Override + public void setManaged(final String id, final boolean managed) { + log.info(String.format("setting managed = '%s' for ds '%s'", managed, id)); + dsRepository.setManaged(id, managed); + apiRepository.setRemovable(id, true); + } + + @Override + public boolean isManaged(final String id) { + return dsRepository.isManaged(id); + } + + @Override + public void updateCompliance(final String dsId, final String apiId, final String compliance, final boolean override) { + log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId)); + apiRepository.updateCompatibility(apiId, compliance); + } + + @Override + public List getApis(final String dsId) { + return apiRepository.findByDatasource(dsId); + } + + @Override + public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException { + final ApiDbEntry api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId)); + try { + if (!api.getRemovable()) { throw new DsmForbiddenException(HttpStatus.SC_UNAUTHORIZED, "api is not removable"); } + + apiRepository.deleteById(apiId); + log.info(String.format("deleted api '%s'", apiId)); + } catch (final EntityNotFoundException e) { + throw new DsmNotFoundException(HttpStatus.SC_NOT_FOUND, "api not found"); + } + } + + @Override + public void addApi(final ApiDbEntry api) { + apiRepository.save(api); + } + + @Override + public boolean existDs(final String dsId) throws DsmException { + return dsRepository.existsById(dsId); + } + + @Override + public void saveDs(final DatasourceDbEntry d) { + log.info(String.format("saving datasource '%s'", d.getId())); + + final DatasourceDbEntry datasource = dsRepository.save(d); + log.info(String.format("saved datasource '%s'", datasource.getId())); + ensureRegistrationDate(d.getId()); + } + + @Override + public void deleteDs(final String dsId) { + dsRepository.deleteById(dsId); + log.info(String.format("deleted datasource '%s'", dsId)); + } + + @Override + public void updateName(final String dsId, final String officialname, final String englishname) { + // TODO what if one of the two names is null or empty? + dsRepository.setDatasourcename(dsId, officialname, englishname); + } + + @Override + public void updateLogoUrl(final String dsId, final String logourl) throws DsmException { + dsRepository.setLogoUrl(dsId, logourl); + } + + @Override + public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) { + dsRepository.setCoordinates(dsId, latitude, longitude); + } + + @Override + public void updateApiBaseUrl(final String apiId, final String baseurl) { + apiRepository.setBaseurl(apiId, baseurl); + } + + @Override + @Transactional + public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException { + final ApiDbEntry api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId)); + if (OAI.equalsIgnoreCase(api.getProtocol())) { + final Set apiParams = api.getApiParams(); + + if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) { + apiRepository.addApiParam(apiId, SET, oaiSet); + log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet)); + return true; + } else { + apiRepository.updateOaiSet(apiId, oaiSet); + log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet)); + return false; + } + } else { + throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol())); + } + } + + @Override + public List findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException { + final PageRequest pageable = PageRequest.of(page, size); + final Specification spec = apiSpec(requestFilter); + final Set set = dsApiRepository.findAll(spec, pageable) + .getContent() + .stream() + .map(DatasourceApiDbEntry::getBaseurl) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toCollection(HashSet::new)); + return Lists.newArrayList(set); + } + + @Override + public void updateTimezone(final String dsId, final String timezone) { + dsRepository.setTimezone(dsId, timezone); + } + + @Override + public void updateEoscDatasourceType(final String dsId, final String type) throws DsmException { + final Vocabulary typologies = vocabularyClient.getDatasourceTypologies(); + if (!typologies.hasCode(type)) { + throw new DsmException( + HttpStatus.SC_BAD_REQUEST, + String.format("invalid datasource type '%s', provide one according to vocabulary %s", type, config.getVocabularies() + .getDatasourceTypologiesEndpoint())); + } + dsRepository.setEoscDatasourceType(dsId, type); + } + + @Override + public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException { + + ensureRegistrationDate(dsId); + + dsRepository.setRegisteringUser(dsId, registeredBy); + + } + + @Override + public void updatePlatform(final String dsId, final String platform) throws DsmException { + dsRepository.setPlatform(dsId, platform); + } + + // HELPER + private void ensureRegistrationDate(final String dsId) { + if (!dsRepository.hasRegistrationdate(dsId)) { + log.info("setting registration date for datasource: " + dsId); + dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis())); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java new file mode 100644 index 00000000..b5adcd8d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceDbEntryRepository.java @@ -0,0 +1,77 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.sql.Date; +import java.util.Optional; + +import javax.transaction.Transactional; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; + +/** + * Created by claudio on 12/04/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public interface DatasourceDbEntryRepository extends JpaRepository, JpaSpecificationExecutor { + + Optional findByNamespaceprefix(String namespaceprefix); + + @Query("select d.managed from #{#entityName} d where d.id = ?1") + boolean isManaged(String id); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.managed = ?2 where d.id = ?1") + void setManaged(String id, boolean managed); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.officialname = ?2, d.englishname = ?3 where d.id = ?1") + void setDatasourcename(String id, String officialname, String englishname); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.logourl = ?2 where d.id = ?1") + void setLogoUrl(String dsId, String logourl); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.latitude = ?2, d.longitude = ?3 where d.id = ?1") + void setCoordinates(String dsId, Double latitude, Double longitude); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.timezone = ?2 where d.id = ?1") + void setTimezone(String dsId, String timezone); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.eoscDatasourceType = ?2 where d.id = ?1") + void setEoscDatasourceType(String dsId, String type); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.registeredby = ?2 where d.id = ?1") + void setRegisteringUser(String id, String registeredby); + + @Query("select case when registrationdate <> null then true else false end as hasregistrationdate from #{#entityName} where id = ?1") + Boolean hasRegistrationdate(String id); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1") + void setRegistrationDate(String id, Date registrationdate); + + @Modifying + @Transactional + @Query("update #{#entityName} d set d.platform = ?2 where d.id = ?1") + void setPlatform(String id, String platform); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java new file mode 100644 index 00000000..79d8e5e8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClient.java @@ -0,0 +1,17 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.Queue; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; + +public interface DatasourceIndexClient { + + IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException; + + String getLastIndexingDate(final IndexDsInfo info) throws DsmException; + + + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java new file mode 100644 index 00000000..659c67a0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceIndexClientImpl.java @@ -0,0 +1,207 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.*; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.*; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.miscutils.functional.hash.Hashing; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; +import eu.dnetlib.openaire.dsm.dao.utils.IndexRecordsInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class DatasourceIndexClientImpl implements DatasourceIndexClient { + + private static final Log log = LogFactory.getLog(DatasourceIndexClientImpl.class); + + public static final String SEPARATOR = "::"; + public static final String DSVERSION = "__dsversion"; + + @Autowired + private DnetOpenaireExporterProperties config; + + private ListeningExecutorService executor; + + private static final Map indexClientMap = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator( + new ScheduledThreadPoolExecutor(5, + new ThreadFactoryBuilder().setNameFormat("datasource-index-client-%d").build())); + } + + @PreDestroy + public void tearDown() { + indexClientMap.forEach((name, client) -> { + try { + client.close(); + } catch (IOException e) { + log.warn(String.format("unable to gracefully shutdown client for index %s", name)); + } + }); + } + + @Override + public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue errors) throws DsmException { + try { + final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR)); + final CloudSolrClient indexClient = getIndexClient(info); + final CountDownLatch latch = new CountDownLatch(2); + final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo(); + + Futures.addCallback( + executor.submit(() -> setDateAndTotal(dsId, collectedFrom, indexClient)), + new FutureCallback() { + + @Override + public void onSuccess(final IndexRecordsInfo info) { + indexRecordInfo + .setTotal(info.getTotal()) + .setDate(info.getDate()); + latch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + latch.countDown(); + } + }, executor); + + Futures.addCallback( + executor.submit(() -> setFunded(dsId, collectedFrom, indexClient)), + new FutureCallback() { + + @Override + public void onSuccess(final Long numFound) { + indexRecordInfo.setFunded(numFound); + latch.countDown(); + } + + @Override + public void onFailure(final Throwable e) { + errors.offer(e); + latch.countDown(); + } + }, executor); + + waitLatch(latch, errors, config.getRequestTimeout()); + return indexRecordInfo; + } catch (final Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("error reading index info", dsId), e); + } + } + + @Override + public String getLastIndexingDate(final IndexDsInfo info) throws DsmException { + try { + final SolrQuery query = new SolrQuery("oaftype:datasource").setRows(1); + final QueryResponse rsp = getIndexClient(info).query(query); + final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null); + final String dsversion = doc.get("__dsversion").toString(); + return StringUtils.substringBefore(dsversion, "T"); + } catch (SolrServerException | IOException e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e); + } + } + + private Long setFunded( + final String dsId, + final String collectedFrom, + final CloudSolrClient indexClient) throws DsmException { + final String query = + String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom); + log.debug(String.format("query: %s", query)); + try { + return indexClient.query(new SolrQuery(query).setRows(0)).getResults().getNumFound(); + } catch (Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for funded results '%s'", dsId), e); + } + } + + private IndexRecordsInfo setDateAndTotal( + final String dsId, + final String collectedFrom, + final CloudSolrClient indexClient) throws DsmException { + try { + final String query = String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom); + log.debug(String.format("query: %s", query)); + + final QueryResponse rsp = indexClient.query(new SolrQuery(query).setRows(1)); + final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument()); + if (log.isDebugEnabled()) { + log.debug(String.format("got document %s", doc.get("__indexrecordidentifier"))); + } + // if (doc.isEmpty()) { + // throw new DatasourceManagerException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching + // query: %s", queryTotal)); + // } + return new IndexRecordsInfo() + .setDate(getDate(doc)) + .setTotal(rsp.getResults().getNumFound()); + } catch (Throwable e) { + throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("Error querying index for date and total '%s'", dsId), e); + } + } + + @SuppressWarnings("unchecked") + private String getDate(final SolrDocument doc) throws DsmException { + final List dsversion = (List) doc.get(DSVERSION); + if (dsversion == null || dsversion.isEmpty()) { throw new DsmException(HttpStatus.INTERNAL_SERVER_ERROR.value(), + String.format("cannot find %s in matched solr document", DSVERSION)); } + final Date date = Iterables.getLast(dsversion); + + return DateFormatUtils.format(date, DsmMappingUtils.DATE_FORMAT); + } + + private synchronized CloudSolrClient getIndexClient(final IndexDsInfo info) { + if (!indexClientMap.containsKey(info.getColl())) { + + final CloudSolrClient client = new Builder(Lists.newArrayList(info.getIndexBaseUrl())).build(); + client.setDefaultCollection(info.getColl()); + + indexClientMap.put(info.getColl(), client); + } + return indexClientMap.get(info.getColl()); + } + + private void waitLatch(final CountDownLatch latch, final Queue errors, final int waitSeconds) { + try { + if (!latch.await(waitSeconds, TimeUnit.SECONDS)) { + errors.offer(new TimeoutException("Waiting for requests to complete has timed out.")); + } + } catch (final InterruptedException e) { + errors.offer(e); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java new file mode 100644 index 00000000..218c527d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java @@ -0,0 +1,141 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; +import eu.dnetlib.openaire.dsm.domain.FilterName; +import eu.dnetlib.openaire.dsm.domain.RequestFilter; +import eu.dnetlib.openaire.dsm.domain.RequestSort; +import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.data.jpa.domain.Specification; + +import javax.persistence.criteria.*; +import java.util.List; +import java.util.Map.Entry; + +public class DatasourceSpecs { + + private static final Log log = LogFactory.getLog(DatasourceSpecs.class); + + public static final String WILDCARD = "%"; + + public static Specification dsRegisteredbyNotNullSpec() { + return (ds, query, cb) -> cb.and( + cb.isNull(ds.get(FilterName.registeredby.name())).not(), + cb.isNull(ds.get("registrationdate")).not()); + } + + public static Specification dsSpec(final RequestSort requestSortBy, final RequestSortOrder order, final RequestFilter requestFilter) { + log.debug(String.format("RequestFilter:'%s', RequestSort:'%s', RequestSortOrder:'%s'", requestFilter, requestSortBy, order)); + return (ds, query, cb) -> { + final Predicate p = cb.conjunction(); + if (requestFilter != null) { + final List> expressions = p.getExpressions(); + requestFilter.entrySet().stream() + .forEach(e -> { + switch (FilterName.type(e.getKey())) { + + case exact: + expressions.add(exactSearch(ds, cb, e)); + + break; + case search: + expressions.add(likeSearch(ds, cb, e)); + + break; + case searchOrgs: + // search by case insensitive organization's country + expressions.add( + cb.equal( + cb.lower( + ds.join("organizations").get(FilterName.country.name())), + getValue(e))); + break; + } + }); + } + if (requestSortBy != null) { + if (order != null) { + final Path orderField = ds.get(requestSortBy.name()); + switch (order) { + case ASCENDING: + query.orderBy(cb.asc(orderField)); + break; + case DESCENDING: + query.orderBy(cb.desc(orderField)); + break; + } + } + } + query.distinct(true); + + return p; + }; + } + + public static Specification apiSpec(final RequestFilter requestFilter) { + log.debug(String.format("RequestFilter:'%s'", requestFilter)); + return (api, query, cb) -> { + final Predicate p = cb.conjunction(); + if (requestFilter != null) { + final List> expressions = p.getExpressions(); + requestFilter.entrySet().stream() + .forEach(e -> { + switch (FilterName.type(e.getKey())) { + + case exact: + expressions.add(exactSearch(api, cb, e)); + break; + case search: + + expressions.add(likeSearch(api, cb, e)); + break; + case searchOrgs: + throw new DsmRuntimeException("not implemented"); + } + }); + } + query.distinct(true); + return p; + }; + } + + // HELPERS + + // substring, case insensitive, like based search + private static Predicate likeSearch(final Root r, final CriteriaBuilder cb, final Entry e) { + return cb.like( + cb.lower( + r.get(e.getKey().name())), + WILDCARD + getValue(e) + WILDCARD); + } + + // search by ID, managed. exact match + private static Predicate exactSearch(final Root r, final CriteriaBuilder cb, final Entry e) { + return cb.equal(r.get(e.getKey().name()), getValue(e)); + } + + private static Object getValue(final Entry e) { + if (e.getValue() instanceof String) { + final String s = ((String) e.getValue()); + + if (!e.getKey().equals(FilterName.country)) { + Boolean b = BooleanUtils.toBooleanObject(s); + if (b != null) { + return b; + } + } + return e.getKey().equals(FilterName.id) ? s : StringUtils.lowerCase(s); + } + if (e.getValue() instanceof Boolean) { + return e.getValue(); + } + + return e.getValue(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java new file mode 100644 index 00000000..511b6240 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.DsmException; + +import java.util.List; + +public interface MongoLoggerClient { + + List getAggregationHistory(final String dsId) throws DsmException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java new file mode 100644 index 00000000..997e936b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java @@ -0,0 +1,234 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.cache.*; +import com.google.common.primitives.Ints; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoClient; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.DnetOpenaireExporterProperties.Datasource; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import eu.dnetlib.enabling.datasources.common.AggregationStage; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.miscutils.datetime.DateUtils; +import eu.dnetlib.openaire.common.Utils; + +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.domain.CollectionInfo; +import eu.dnetlib.openaire.dsm.domain.CollectionMode; +import eu.dnetlib.openaire.dsm.domain.TransformationInfo; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpStatus; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import static com.mongodb.client.model.Filters.*; +import static com.mongodb.client.model.Projections.fields; +/** + * Created by claudio on 20/10/2016. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class MongoLoggerClientImpl implements MongoLoggerClient { + + private static final Log log = LogFactory.getLog(MongoLoggerClientImpl.class); + + @Autowired + private MongoClient datasourcePublisherMongoClient; + + @Autowired + private DnetOpenaireExporterProperties config; + + private final static String LOADTIME = "loadtime"; + private final LoadingCache loadingCache = CacheBuilder.newBuilder() + .maximumSize(1) + .expireAfterWrite(60, TimeUnit.MINUTES) + .build(new CacheLoader() { + // The only cached value is associated to "loadtime" + public Instant load(String key) { + final Instant loadTime = getLoadTime(); + log.debug("found load time: " + loadTime.toString()); + return loadTime; + } + }); + + + private static final Bson fields = getFields(); + + private static MongoCollection collection = null; + + @Override + @Cacheable("dsm-aggregationhistory-cache") + public List getAggregationHistory(final String dsId) throws DsmException { + log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId)); + final Datasource conf = config.getDatasource(); + try { + final FindIterable aggregationDocs = getCollection().find(queryForAggregationHistory(dsId, "(collect|transform)")) + .projection(fields) + .limit(conf.getMongoQueryLimit()) + .sort(dbo("system:startHumanDate", -1)); + + final List aggregationInfos = Utils.stream(aggregationDocs.iterator()) + .map(getMapper()) + .filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate())) + .collect(Collectors.toList()); + + final Instant loadTime = loadingCache.get(LOADTIME); + + if (!Objects.equals(Instant.MIN, loadTime)) { + for (final AggregationInfo a : aggregationInfos) { + if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) { + a.setIndexedVersion(true); + break; + } + } + } + + return aggregationInfos; + } catch (Throwable e) { + throw new DsmException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e); + } + } + + private Instant getLoadTime() { + log.warn("querying for metadata load time, not using cache"); + return Optional.ofNullable(getCollection().find(queryForLastMetadataLoad())) + .map(d -> d.sort(dbo("system:startHumanDate", -1)).first()) + .map(d -> (String) d.getOrDefault("system:startHumanDate", "")) + .map(s -> Instant.parse(s.replaceAll("\\+.*", "Z"))) + .orElse(Instant.MIN); + } + + private Instant asInstant(final AggregationInfo a) { + return Instant.parse(a.getDate() + "T00:00:00Z"); + } + + @Override + @CacheEvict(cacheNames = { "dsm-aggregationhistory-cache", "dsm-firstharvestdate-cache" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager aggregation history cache"); + } + + private Function getMapper() { + return new Function() { + + @Override + public AggregationInfo apply(final Document d) { + + AggregationInfo info = null; + final AggregationStage stage = AggregationStage.parse(d.getString("system:wfName")); + switch (stage) { + + case COLLECT: + CollectionInfo cInfo = new CollectionInfo(); + cInfo.setAggregationStage(stage); + cInfo.setCollectionMode(getCollectionMode(d)); + cInfo.setNumberOfRecords(getNumberOfRecords(d)); + cInfo.setDate(getDate(d)); + info = cInfo; + break; + case TRANSFORM: + TransformationInfo tInfo = new TransformationInfo(); + tInfo.setAggregationStage(stage); + tInfo.setNumberOfRecords(getNumberOfRecords(d)); + tInfo.setDate(getDate(d)); + info = tInfo; + break; + } + return info; + } + + private CollectionMode getCollectionMode(Document d) { + return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection")) + .map(CollectionMode::valueOf) + .orElseGet(() -> + Optional.ofNullable(d.getString("collectionMode")) + .map(CollectionMode::valueOf) + .orElse(null)); + } + + private Integer getNumberOfRecords(final Document d) { + final String sinkSize = d.getString("mainlog:sinkSize"); + final String total = d.getString("mainlog:total"); + + if (StringUtils.isNotBlank(sinkSize)) { + return Ints.tryParse(sinkSize); + } else if (StringUtils.isNotBlank(total)) { + return Ints.tryParse(total); + } else { + return -1; + } + } + }; + } + + private String getDate(final Document d) { + final String dateString = d.getString("system:startHumanDate"); + if (StringUtils.isBlank(dateString)) { return ""; } + return DateFormatUtils.format(new DateUtils().parse(dateString), DsmMappingUtils.DATE_FORMAT); + } + + private static Bson getFields() { + return fields( + eq("system:wfName", 1), + eq("system:node:SELECT_MODE:selection", 1), + eq("collectionMode", 1), + eq("mainlog:sinkSize", 1), + eq("mainlog:writeOps", 1), + eq("mainlog:total", 1), + eq("system:startHumanDate", 1), + eq("system:profileName", 1)); + } + + private static BasicDBObject dbo(final String key, final Object value) { + return new BasicDBObject(key, value); + } + + private Bson queryForAggregationHistory(final String dsId, final String pattern) { + return and( + eq("parentDatasourceId", dsId), + eq("system:profileFamily", "aggregator"), + eq("system:isCompletedSuccessfully", "true"), + regex("system:wfName", pattern, "i")); + } + + private Bson queryForLastMetadataLoad() { + try { + final String contentLoadQuery = config.getContentLoadQuery(); + log.debug("parsing content load query: " + contentLoadQuery); + return new ObjectMapper().readValue(contentLoadQuery, BasicDBObject.class); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private synchronized MongoCollection getCollection() { + if (collection == null) { + log.info("inizializing mongodb collection ..."); + final Datasource conf = config.getDatasource(); + collection = datasourcePublisherMongoClient.getDatabase(conf.getMongoDbName()).getCollection(conf.getMongoCollectionName()); + } + return collection; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java new file mode 100644 index 00000000..131fa713 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClient.java @@ -0,0 +1,9 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmException; + +public interface ObjectStoreClient { + + Long getObjectStoreSize(final String objectStoreId) throws DsmException; + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java new file mode 100644 index 00000000..78763134 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ObjectStoreClientImpl.java @@ -0,0 +1,37 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.data.objectstore.rmi.ObjectStoreService; +import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException; +import eu.dnetlib.enabling.datasources.common.DsmException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +public class ObjectStoreClientImpl implements ObjectStoreClient { + + private static final Log log = LogFactory.getLog(ObjectStoreClientImpl.class); + + @Autowired + private ObjectStoreService objectStoreService; + + @Override + public Long getObjectStoreSize(final String objectStoreId) throws DsmException { + log.debug("get size for objectStore " + objectStoreId); + if (StringUtils.isBlank(objectStoreId)) { + return 0L; + } + try { + final long size = objectStoreService.getSize(objectStoreId); + log.debug("got objectStore size: " + size); + return size; + } catch (ObjectStoreServiceException e) { + throw new DsmException("unable to get size for objectStore " + objectStoreId); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java new file mode 100644 index 00000000..a2d422b8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java @@ -0,0 +1,61 @@ +package eu.dnetlib.openaire.dsm.dao; + +import java.util.List; +import java.util.Queue; + +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.DatasourceSearchResponse; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.Header; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; + +public class ResponseUtils { + + public static ApiDetailsResponse apiResponse(final List api, final long total) { + final ApiDetailsResponse rsp = new ApiDetailsResponse().setApi(api); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceSnippetResponse snippetResponse(final List snippets, final long total) { + final DatasourceSnippetResponse rsp = new DatasourceSnippetResponse(snippets); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceDetailResponse detailsResponse(final List details, final long total) { + final DatasourceDetailResponse rsp = new DatasourceDetailResponse(details); + rsp.setHeader(header(total)); + return rsp; + } + + public static DatasourceSearchResponse searchResponse(final List infos, final long total) { + final DatasourceSearchResponse rsp = new DatasourceSearchResponse(infos); + rsp.setHeader(header(total)); + return rsp; + } + + public static Header header(final Queue errors, final long total) { + return Header.newInsance() + .setExceptions(errors) + .setTotal(total); + } + + public static Header header(final long total) { + return header(Lists.newLinkedList(), total); + } + + public static SimpleResponse simpleResponse(final List list) { + final SimpleResponse rsp = new SimpleResponse().setResponse(list);; + rsp.setHeader(header(Lists.newLinkedList(), list.size())); + return rsp; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java new file mode 100644 index 00000000..2ed99c55 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClient.java @@ -0,0 +1,14 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.vocabularies.Vocabulary; + +public interface VocabularyClient { + + Vocabulary getCountries() throws DsmException; + + Vocabulary getDatasourceTypologies() throws DsmException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java new file mode 100644 index 00000000..3a9ad1b8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/VocabularyClientImpl.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.dsm.dao; + +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.vocabularies.Vocabulary; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +/** + * Created by claudio on 15/09/2017. + */ +@Component +public class VocabularyClientImpl implements VocabularyClient { + + private static final Log log = LogFactory.getLog(VocabularyClientImpl.class); + + @Autowired + private DnetOpenaireExporterProperties config; + + @Override + @Cacheable("vocabularies-cache") + public Vocabulary getCountries() throws DsmException { + return _getVocabulary(config.getVocabularies().getCountriesEndpoint(), Vocabulary.class); + } + + @Override + @Cacheable("vocabularies-cache") + public Vocabulary getDatasourceTypologies() throws DsmException { + return _getVocabulary(config.getVocabularies().getDatasourceTypologiesEndpoint(), Vocabulary.class); + } + + private T _getVocabulary(final String endpoint, Class clazz) throws DsmException { + final RestTemplate rt = new RestTemplate(); + log.info("get vocabulary from " + endpoint); + final ResponseEntity rsp = rt.getForEntity(endpoint, clazz); + + if (!rsp.getStatusCode().is2xxSuccessful()) { + throw new DsmException(rsp.getStatusCodeValue(), "unable to read content from " + endpoint); + } + + return rsp.getBody(); + } + + @Override + @CacheEvict(cacheNames = "vocabularies-cache", allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped dsManager vocabulary cache"); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java new file mode 100644 index 00000000..86ce80cb --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/DsmMappingUtils.java @@ -0,0 +1,180 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +import java.sql.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.BeanWrapperImpl; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions; +import eu.dnetlib.openaire.dsm.domain.ApiDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; +import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended; +import eu.dnetlib.openaire.dsm.domain.OrganizationDetails; +import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.dsm.domain.db.OrganizationDbEntry; + +public class DsmMappingUtils { + + public static final String DATE_FORMAT = "yyyy-MM-dd"; + + public static final String ID_SEPARATOR = "::"; + + public static final String ID_PREFIX = "api_________" + ID_SEPARATOR; + + public static String createId(final ApiDetails api) { + return ID_PREFIX + api.getDatasource() + ID_SEPARATOR + RandomStringUtils.randomAlphanumeric(8); + } + + public static DatasourceDetails asDetails(final DatasourceDbEntry d) { + final DatasourceDetails details = _convert(d, DatasourceDetails.class); + return details.setOpenaireId(asOpenaireId(details.getId())); + } + + public static DatasourceSnippetExtended asSnippetExtended(final DatasourceDbEntry d) { + final DatasourceSnippetExtended ds = new DatasourceSnippetExtended(); + ds.setId(d.getId()); + ds.setOfficialname(d.getOfficialname()); + ds.setEnglishname(d.getEnglishname()); + ds.setRegisteredby(d.getRegisteredby()); + ds.setWebsiteurl(d.getWebsiteurl()); + ds.setEoscDatasourceType(d.getEoscDatasourceType()); + ds.setRegistrationdate(d.getRegistrationdate()); + ds.setLogoUrl(d.getLogourl()); + ds.setDescription(d.getDescription()); + ds.setConsentTermsOfUse(d.getConsentTermsOfUse()); + ds.setConsentTermsOfUseDate(d.getConsentTermsOfUseDate()); + ds.setLastConsentTermsOfUseDate(d.getLastConsentTermsOfUseDate()); + ds.setFullTextDownload(d.getFullTextDownload()); + if (d.getOrganizations() != null) { + ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet())); + } + ds.setTypology(d.getTypology()); + return ds; + } + + private static OrganizationDetails asOrganizationDetail(final OrganizationDbEntry o) { + return new OrganizationDetails() + .setCountry(o.getCountry()) + .setLegalname(o.getLegalname()) + .setLegalshortname(o.getLegalshortname()) + .setWebsiteurl(o.getWebsiteurl()) + .setLogourl(o.getLogourl()); + } + + public static ApiDetails asDetails(final ApiDbEntry d) { + return _convert(d, ApiDetails.class); + } + + public static ApiDbEntry asDbEntry(final ApiDetails d) { + final ApiDbEntry apiDbEntry = _convert(d, ApiDbEntry.class); + + // Need to complete the references among objects, because you know, referential integrity ... + apiDbEntry.getApiParams().forEach(ap -> ap.getId().setApi(apiDbEntry)); + + return apiDbEntry; + } + + public static DatasourceDbEntry asDbEntry(final DatasourceDetails d) { + final DatasourceDbEntry dbe = _convert(d, DatasourceDbEntry.class); + if (dbe.getOrganizations() != null) { + dbe.getOrganizations().forEach(o -> { + final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); + o.setId(prefix + ID_SEPARATOR + o.getLegalname()); + if (o.getDateofcollection() == null) { + o.setDateofcollection(new Date(System.currentTimeMillis())); + } + o.setCollectedfrom(dbe.getCollectedfrom()); + }); + } + + _fix_typology(dbe, d); + + return dbe; + } + + @Deprecated + private static void _fix_typology(final DatasourceDbEntry dbe, final DatasourceDetails d) { + if (StringUtils.isNotBlank(d.getTypology()) && StringUtils.isBlank(d.getEoscDatasourceType())) { + // THE ORDER IS IMPORTANT: DO NOT CHANGE IT + if (d.getTypology().startsWith("crissystem")) { + dbe.setEoscDatasourceType("CRIS system"); + } else if (d.getTypology().startsWith("entityregistry")) { + dbe.setEoscDatasourceType("Registry"); + } else if (d.getTypology().startsWith("pubscatalogue") || d.getTypology().equals("websource")) { + dbe.setEoscDatasourceType("Catalogue"); + } else if (d.getTypology().contains("journal")) { + dbe.setEoscDatasourceType("Journal archive"); + } else if (d.getTypology().startsWith("aggregator")) { + dbe.setEoscDatasourceType("Aggregator"); + } else if (d.getTypology().contains("repository")) { + dbe.setEoscDatasourceType("Repository"); + } else { + dbe.setEoscDatasourceType("Aggregator"); + } + } else if (StringUtils.isBlank(d.getTypology()) && StringUtils.isNotBlank(d.getEoscDatasourceType())) { + if (dbe.getEoscDatasourceType().equals("CRIS system")) { + dbe.setTypology("crissystem"); + } else if (dbe.getEoscDatasourceType().equals("Registry")) { + dbe.setTypology("entityregistry"); + } else if (dbe.getEoscDatasourceType().equals("Catalogue")) { + dbe.setTypology("pubscatalogue::unknown"); + } else if (dbe.getEoscDatasourceType().equals("Journal archive")) { + dbe.setTypology("pubsrepository::journal"); + } else if (dbe.getEoscDatasourceType().equals("Aggregator")) { + dbe.setTypology("aggregator"); + } else if (dbe.getEoscDatasourceType().equals("Repository")) { + dbe.setTypology("pubsrepository::unknown"); + } else { + dbe.setTypology("aggregator"); + } + } + } + + public static DatasourceDbEntry asDbEntry(final DatasourceDetailsUpdate d) { + return _convert(d, DatasourceDbEntry.class); + } + + // HELPERS + + private static T _convert(final Object o, final Class clazz) { + final ObjectMapper mapper = new ObjectMapper(); + return mapper.convertValue(o, clazz); + } + + private static String asOpenaireId(final String id) { + final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR); + final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR); + + return prefix + ID_SEPARATOR + AbstractDNetXsltFunctions.md5(md5); + } + + public static void copyNonNullProperties(final Object src, final Object target) { + BeanUtils.copyProperties(src, target, getNullPropertyNames(src)); + } + + public static String[] getNullPropertyNames(final Object source) { + final BeanWrapper src = new BeanWrapperImpl(source); + final java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors(); + + final Set emptyNames = new HashSet<>(); + for (final java.beans.PropertyDescriptor pd : pds) { + final Object srcValue = src.getPropertyValue(pd.getName()); + if (srcValue == null) { + emptyNames.add(pd.getName()); + } + } + final String[] result = new String[emptyNames.size()]; + return emptyNames.toArray(result); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java new file mode 100644 index 00000000..effb9491 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexDsInfo.java @@ -0,0 +1,36 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +/** + * Created by claudio on 20/10/2016. + */ +public class IndexDsInfo { + + private final String indexBaseUrl; + private final String indexDsId; + private final String format; + private final String coll; + + public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) { + this.indexBaseUrl = indexBaseUrl; + this.indexDsId = indexDsId; + this.format = format; + this.coll = coll; + } + + public String getIndexBaseUrl() { + return indexBaseUrl; + } + + public String getIndexDsId() { + return indexDsId; + } + + public String getFormat() { + return format; + } + + public String getColl() { + return coll; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java new file mode 100644 index 00000000..5f3e1256 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/utils/IndexRecordsInfo.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.dsm.dao.utils; + +/** + * Created by claudio on 21/10/2016. + */ +public class IndexRecordsInfo { + + private long total; + + private long funded; + + private String date; + + public IndexRecordsInfo() {} + + public IndexRecordsInfo(final long total, final long funded, final String date) { + this.total = total; + this.funded = funded; + this.date = date; + } + + public long getTotal() { + return total; + } + + public IndexRecordsInfo setTotal(final long total) { + this.total = total; + return this; + } + + public long getFunded() { + return funded; + } + + public IndexRecordsInfo setFunded(final long funded) { + this.funded = funded; + return this; + } + + public String getDate() { + return date; + } + + public IndexRecordsInfo setDate(final String date) { + this.date = date; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java new file mode 100644 index 00000000..eeddbc94 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class AggregationHistoryResponse extends Response { + + @ApiModelProperty(position = 1) + private List aggregationInfo; + + public AggregationHistoryResponse(List aggregationInfo) { + super(); + this.aggregationInfo = aggregationInfo; + } + + public List getAggregationInfo() { + return aggregationInfo; + } + + public void setAggregationInfo(List aggregationInfo) { + this.aggregationInfo = aggregationInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java new file mode 100644 index 00000000..d2ab101a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetails.java @@ -0,0 +1,217 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Api model", description = "provides information about the datasource API") +public class ApiDetails extends ApiIgnoredProperties { + + @ApiModelProperty(position = 0) + private String id = null; + + @ApiModelProperty(position = 1) + private String protocol = null; + + @ApiModelProperty(position = 2) + private String datasource = null; + + @ApiModelProperty(position = 3) + private String contentdescription = null; + + @ApiModelProperty(position = 4) + private String eoscDatasourceType = null; + + @ApiModelProperty(position = 5) + private String compatibility; + + @ApiModelProperty(position = 7) + private String compatibilityOverride; + + @ApiModelProperty(position = 8) + private Integer lastCollectionTotal; + + @ApiModelProperty(position = 9) + private Date lastCollectionDate; + + @ApiModelProperty(position = 10) + private Integer lastAggregationTotal; + + @ApiModelProperty(position = 11) + private Date lastAggregationDate; + + @ApiModelProperty(position = 12) + private Integer lastDownloadTotal; + + @ApiModelProperty(position = 13) + private Date lastDownloadDate; + + @ApiModelProperty(position = 14) + private String baseurl; + + @ApiModelProperty(position = 15) + protected Boolean removable = false; + + @ApiModelProperty(position = 16) + private Set apiParams; + + @ApiModelProperty(position = 17) + private String metadataIdentifierPath = ""; + + public String getId() { + return id; + } + + public String getProtocol() { + return protocol; + } + + public String getDatasource() { + return datasource; + } + + public String getContentdescription() { + return contentdescription; + } + + public String getCompatibility() { + return compatibility; + } + + public Integer getLastCollectionTotal() { + return lastCollectionTotal; + } + + public Date getLastCollectionDate() { + return lastCollectionDate; + } + + public Integer getLastAggregationTotal() { + return lastAggregationTotal; + } + + public Date getLastAggregationDate() { + return lastAggregationDate; + } + + public Integer getLastDownloadTotal() { + return lastDownloadTotal; + } + + public Date getLastDownloadDate() { + return lastDownloadDate; + } + + public String getBaseurl() { + return baseurl; + } + + public ApiDetails setId(final String id) { + this.id = id; + return this; + } + + public ApiDetails setProtocol(final String protocol) { + this.protocol = protocol; + return this; + } + + public ApiDetails setDatasource(final String datasource) { + this.datasource = datasource; + return this; + } + + public ApiDetails setContentdescription(final String contentdescription) { + this.contentdescription = contentdescription; + return this; + } + + public ApiDetails setCompatibility(final String compatibility) { + this.compatibility = compatibility; + return this; + } + + public ApiDetails setLastCollectionTotal(final Integer lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + return this; + } + + public ApiDetails setLastCollectionDate(final Date lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + return this; + } + + public ApiDetails setLastAggregationTotal(final Integer lastAggregationTotal) { + this.lastAggregationTotal = lastAggregationTotal; + return this; + } + + public ApiDetails setLastAggregationDate(final Date lastAggregationDate) { + this.lastAggregationDate = lastAggregationDate; + return this; + } + + public ApiDetails setLastDownloadTotal(final Integer lastDownloadTotal) { + this.lastDownloadTotal = lastDownloadTotal; + return this; + } + + public ApiDetails setLastDownloadDate(final Date lastDownloadDate) { + this.lastDownloadDate = lastDownloadDate; + return this; + } + + public ApiDetails setBaseurl(final String baseurl) { + this.baseurl = baseurl; + return this; + } + + public Set getApiParams() { + return apiParams; + } + + public void setApiParams(final Set apiParams) { + this.apiParams = apiParams; + } + + public String getCompatibilityOverride() { + return compatibilityOverride; + } + + public ApiDetails setCompatibilityOverride(final String compatibilityOverride) { + this.compatibilityOverride = compatibilityOverride; + return this; + } + + public Boolean getRemovable() { + return removable; + } + + public ApiDetails setRemovable(final Boolean removable) { + this.removable = removable; + return this; + } + + public String getMetadataIdentifierPath() { + return metadataIdentifierPath; + } + + public ApiDetails setMetadataIdentifierPath(final String metadataIdentifierPath) { + this.metadataIdentifierPath = metadataIdentifierPath; + return this; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public ApiDetails setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java new file mode 100644 index 00000000..495dc908 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiDetailsResponse.java @@ -0,0 +1,24 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class ApiDetailsResponse extends Response { + + @ApiModelProperty(position = 1) + private List api; + + public List getApi() { + return api; + } + + public ApiDetailsResponse setApi(final List api) { + this.api = api; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java new file mode 100644 index 00000000..d674f260 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiIgnoredProperties.java @@ -0,0 +1,72 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public abstract class ApiIgnoredProperties { + + @JsonIgnore + protected Boolean active = false; + + @JsonIgnore + protected String lastCollectionMdid; + + @JsonIgnore + protected String lastAggregationMdid; + + @JsonIgnore + protected String lastDownloadObjid; + + @JsonIgnore + protected String lastValidationJob; + + @JsonIgnore + protected boolean compatibilityOverrided; + + public Boolean getActive() { + return active; + } + + public void setActive(final Boolean active) { + this.active = active; + } + + public String getLastCollectionMdid() { + return lastCollectionMdid; + } + + public void setLastCollectionMdid(final String lastCollectionMdid) { + this.lastCollectionMdid = lastCollectionMdid; + } + + public String getLastAggregationMdid() { + return lastAggregationMdid; + } + + public void setLastAggregationMdid(final String lastAggregationMdid) { + this.lastAggregationMdid = lastAggregationMdid; + } + + public String getLastDownloadObjid() { + return lastDownloadObjid; + } + + public void setLastDownloadObjid(final String lastDownloadObjid) { + this.lastDownloadObjid = lastDownloadObjid; + } + + public String getLastValidationJob() { + return lastValidationJob; + } + + public void setLastValidationJob(final String lastValidationJob) { + this.lastValidationJob = lastValidationJob; + } + + public boolean isCompatibilityOverrided() { + return compatibilityOverrided; + } + + public void setCompatibilityOverrided(final boolean compatibilityOverrided) { + this.compatibilityOverrided = compatibilityOverrided; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java new file mode 100644 index 00000000..158bd72a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/ApiParamDetails.java @@ -0,0 +1,24 @@ +package eu.dnetlib.openaire.dsm.domain; + +public class ApiParamDetails { + + protected String param; + + protected String value; + + public String getParam() { + return param; + } + + public void setParam(final String param) { + this.param = param; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java new file mode 100644 index 00000000..60df410d --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 29/11/2016. + */ +@ApiModel +@JsonAutoDetect +public class CollectionInfo extends AggregationInfo { + + private CollectionMode collectionMode; + + public CollectionMode getCollectionMode() { + return collectionMode; + } + + public void setCollectionMode(final CollectionMode collectionMode) { + this.collectionMode = collectionMode; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java new file mode 100644 index 00000000..996bd6bf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionMode.java @@ -0,0 +1,13 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 12/09/16. + */ +@ApiModel +@JsonAutoDetect +public enum CollectionMode { + REFRESH, INCREMENTAL +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java new file mode 100644 index 00000000..75ca070b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceDetailResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceDetailResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java new file mode 100644 index 00000000..5285fb6e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetails.java @@ -0,0 +1,441 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import javax.persistence.Transient; +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * Created by claudio on 12/09/16. + */ +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides information about the datasource") +public class DatasourceDetails extends DatasourceIgnoredProperties { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @Transient + @ApiModelProperty(position = 1) + private String openaireId; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @ApiModelProperty(position = 5) + private String logourl; + + @Email + @ApiModelProperty(position = 6) + private String contactemail; + + @ApiModelProperty(position = 7) + private Double latitude; + + @ApiModelProperty(position = 8) + private Double longitude; + + @ApiModelProperty(position = 9) + private String timezone; + + @NotBlank + @ApiModelProperty(position = 10) + private String namespaceprefix; + + @ApiModelProperty(position = 11) + private String languages; + + @ApiModelProperty(position = 12) + private Date dateofvalidation; + + @NotBlank + @ApiModelProperty(position = 13) + private String eoscDatasourceType; + + @ApiModelProperty(position = 14) + private Date dateofcollection; + + @ApiModelProperty(position = 15) + private String platform; + + @ApiModelProperty(position = 16) + private String activationId; + + @ApiModelProperty(position = 17) + private String description; + + @ApiModelProperty(position = 18) + private String issn; + + @ApiModelProperty(position = 19) + private String eissn; + + @ApiModelProperty(position = 20) + private String lissn; + + @Email + @ApiModelProperty(position = 21) + private String registeredby; + + @ApiModelProperty(position = 22) + private String subjects; + + @ApiModelProperty(position = 23) + protected String aggregator = "OPENAIRE"; + + @ApiModelProperty(position = 24) + protected String collectedfrom; + + @ApiModelProperty(position = 25) + private Boolean managed; + + @ApiModelProperty(position = 28) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 29) + private Boolean fullTextDownload; + + @ApiModelProperty(position = 30) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 31) + private Date lastConsentTermsOfUseDate; + + @ApiModelProperty(position = 26) + private Set organizations; + + @ApiModelProperty(position = 27) + private Set identities; + + @ApiModelProperty(position = 32) + private String status; + + @Deprecated + @ApiModelProperty(position = 33) + private String typology; + + public String getId() { + return id; + } + + public String getOpenaireId() { + return openaireId; + } + + public String getOfficialname() { + return officialname; + } + + public String getEnglishname() { + return englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getContactemail() { + return contactemail; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public String getTimezone() { + return timezone; + } + + public String getLanguages() { + return languages; + } + + public String getNamespaceprefix() { + return namespaceprefix; + } + + public Date getDateofvalidation() { + return dateofvalidation; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public Date getDateofcollection() { + return dateofcollection; + } + + public String getPlatform() { + return platform; + } + + public String getActivationId() { + return activationId; + } + + public String getDescription() { + return description; + } + + public String getIssn() { + return issn; + } + + public String getEissn() { + return eissn; + } + + public String getLissn() { + return lissn; + } + + public String getRegisteredby() { + return registeredby; + } + + public String getSubjects() { + return subjects; + } + + public String getAggregator() { + return aggregator; + } + + public String getCollectedfrom() { + return collectedfrom; + } + + public Boolean getManaged() { + return managed; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public Set getOrganizations() { + return organizations; + } + + public Set getIdentities() { + return identities; + } + + public DatasourceDetails setId(final String id) { + this.id = id; + return this; + } + + public DatasourceDetails setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + return this; + } + + public DatasourceDetails setOfficialname(final String officialname) { + this.officialname = officialname; + return this; + } + + public DatasourceDetails setEnglishname(final String englishname) { + this.englishname = englishname; + return this; + } + + public DatasourceDetails setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public DatasourceDetails setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public DatasourceDetails setContactemail(final String contactemail) { + this.contactemail = contactemail; + return this; + } + + public DatasourceDetails setLatitude(final Double latitude) { + this.latitude = latitude; + return this; + } + + public DatasourceDetails setLongitude(final Double longitude) { + this.longitude = longitude; + return this; + } + + public DatasourceDetails setTimezone(final String timezone) { + this.timezone = timezone; + return this; + } + + public DatasourceDetails setLanguages(final String languages) { + this.languages = languages; + return this; + } + + public DatasourceDetails setNamespaceprefix(final String namespaceprefix) { + this.namespaceprefix = namespaceprefix; + return this; + } + + public DatasourceDetails setDateofvalidation(final Date dateofvalidation) { + this.dateofvalidation = dateofvalidation; + return this; + } + + public DatasourceDetails setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + return this; + } + + public DatasourceDetails setDateofcollection(final Date dateofcollection) { + this.dateofcollection = dateofcollection; + return this; + } + + public DatasourceDetails setPlatform(final String platform) { + this.platform = platform; + return this; + } + + public DatasourceDetails setActivationId(final String activationId) { + this.activationId = activationId; + return this; + } + + public DatasourceDetails setDescription(final String description) { + this.description = description; + return this; + } + + public DatasourceDetails setIssn(final String issn) { + this.issn = issn; + return this; + } + + public DatasourceDetails setEissn(final String eissn) { + this.eissn = eissn; + return this; + } + + public DatasourceDetails setLissn(final String lissn) { + this.lissn = lissn; + return this; + } + + public DatasourceDetails setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + return this; + } + + public DatasourceDetails setSubjects(final String subjects) { + this.subjects = subjects; + return this; + } + + public DatasourceDetails setAggregator(final String aggregator) { + this.aggregator = aggregator; + return this; + } + + public DatasourceDetails setCollectedfrom(final String collectedfrom) { + this.collectedfrom = collectedfrom; + return this; + } + + public DatasourceDetails setManaged(final Boolean managed) { + this.managed = managed; + return this; + } + + public DatasourceDetails setOrganizations(final Set organizations) { + this.organizations = organizations; + return this; + } + + public DatasourceDetails setIdentities(final Set identities) { + this.identities = identities; + return this; + } + + public DatasourceDetails setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + return this; + } + + public DatasourceDetails setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + return this; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; + } + + public String getStatus() { + return status; + } + + public DatasourceDetails setStatus(final String status) { + this.status = status; + return this; + } + + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public DatasourceDetails setTypology(final String typology) { + this.typology = typology; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java new file mode 100644 index 00000000..311773a9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceDetailsUpdate.java @@ -0,0 +1,255 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; +import java.util.Set; + +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +/** + * Created by claudio on 12/09/16. + */ +@JsonAutoDetect +@ApiModel(value = "Datasource updatable fields model", description = "provides information about the datasource field that can be updated") +public class DatasourceDetailsUpdate { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @ApiModelProperty(position = 5) + private String logourl; + + @Email + @ApiModelProperty(position = 6) + private String contactemail; + + @ApiModelProperty(position = 7) + private Double latitude; + + @ApiModelProperty(position = 8) + private Double longitude; + + @ApiModelProperty(position = 9) + private String timezone; + + @NotBlank + @ApiModelProperty(position = 13) + private String eoscDatasourceType; + + @ApiModelProperty(position = 15) + private String platform; + + @ApiModelProperty(position = 17) + private String description; + + @Email + @ApiModelProperty(position = 21) + private String registeredby; + + @ApiModelProperty(position = 25) + private Boolean managed; + + @ApiModelProperty(position = 27) + private Set identities; + + @ApiModelProperty(position = 28) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 29) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 29) + private Date lastConsentTermsOfUseDate; + + @ApiModelProperty(position = 31) + private Boolean fullTextDownload; + + public String getId() { + return id; + } + + public String getOfficialname() { + return officialname; + } + + public String getEnglishname() { + return englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getContactemail() { + return contactemail; + } + + public Double getLatitude() { + return latitude; + } + + public Double getLongitude() { + return longitude; + } + + public String getTimezone() { + return timezone; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public String getPlatform() { + return platform; + } + + public String getDescription() { + return description; + } + + public String getRegisteredby() { + return registeredby; + } + + public Boolean getManaged() { + return managed; + } + + public Set getIdentities() { + return identities; + } + + public DatasourceDetailsUpdate setId(final String id) { + this.id = id; + return this; + } + + public DatasourceDetailsUpdate setOfficialname(final String officialname) { + this.officialname = officialname; + return this; + } + + public DatasourceDetailsUpdate setEnglishname(final String englishname) { + this.englishname = englishname; + return this; + } + + public DatasourceDetailsUpdate setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public DatasourceDetailsUpdate setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public DatasourceDetailsUpdate setContactemail(final String contactemail) { + this.contactemail = contactemail; + return this; + } + + public DatasourceDetailsUpdate setLatitude(final Double latitude) { + this.latitude = latitude; + return this; + } + + public DatasourceDetailsUpdate setLongitude(final Double longitude) { + this.longitude = longitude; + return this; + } + + public DatasourceDetailsUpdate setTimezone(final String timezone) { + this.timezone = timezone; + return this; + } + + public DatasourceDetailsUpdate setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + return this; + } + + public DatasourceDetailsUpdate setPlatform(final String platform) { + this.platform = platform; + return this; + } + + public DatasourceDetailsUpdate setDescription(final String description) { + this.description = description; + return this; + } + + public DatasourceDetailsUpdate setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + return this; + } + + public DatasourceDetailsUpdate setManaged(final Boolean managed) { + this.managed = managed; + return this; + } + + public DatasourceDetailsUpdate setIdentities(final Set identities) { + this.identities = identities; + return this; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public DatasourceDetailsUpdate setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + return this; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public DatasourceDetailsUpdate setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java new file mode 100644 index 00000000..94374a5f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceIgnoredProperties.java @@ -0,0 +1,196 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.sql.Date; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public abstract class DatasourceIgnoredProperties { + + @JsonIgnore + protected String od_contenttypes; + + @JsonIgnore + protected String provenanceaction; + + @JsonIgnore + protected Date releasestartdate; + + @JsonIgnore + protected Date releaseenddate; + + @JsonIgnore + protected String missionstatementurl; + + @JsonIgnore + protected Boolean dataprovider; + + @JsonIgnore + protected Boolean serviceprovider; + + @JsonIgnore + protected String databaseaccesstype; + + @JsonIgnore + protected String datauploadtype; + + @JsonIgnore + protected String databaseaccessrestriction; + + @JsonIgnore + protected String datauploadrestriction; + + @JsonIgnore + protected Boolean versioning; + + @JsonIgnore + protected String citationguidelineurl; + + @JsonIgnore + protected String qualitymanagementkind; + + @JsonIgnore + protected String pidsystems; + + @JsonIgnore + protected String certificates; + + @JsonIgnore + protected Date registrationdate; + + public String getOd_contenttypes() { + return od_contenttypes; + } + + public void setOd_contenttypes(final String od_contenttypes) { + this.od_contenttypes = od_contenttypes; + } + + public String getProvenanceaction() { + return provenanceaction; + } + + public void setProvenanceaction(final String provenanceaction) { + this.provenanceaction = provenanceaction; + } + + public Date getReleasestartdate() { + return releasestartdate; + } + + public void setReleasestartdate(final Date releasestartdate) { + this.releasestartdate = releasestartdate; + } + + public Date getReleaseenddate() { + return releaseenddate; + } + + public void setReleaseenddate(final Date releaseenddate) { + this.releaseenddate = releaseenddate; + } + + public String getMissionstatementurl() { + return missionstatementurl; + } + + public void setMissionstatementurl(final String missionstatementurl) { + this.missionstatementurl = missionstatementurl; + } + + public Boolean getDataprovider() { + return dataprovider; + } + + public void setDataprovider(final Boolean dataprovider) { + this.dataprovider = dataprovider; + } + + public Boolean getServiceprovider() { + return serviceprovider; + } + + public void setServiceprovider(final Boolean serviceprovider) { + this.serviceprovider = serviceprovider; + } + + public String getDatabaseaccesstype() { + return databaseaccesstype; + } + + public void setDatabaseaccesstype(final String databaseaccesstype) { + this.databaseaccesstype = databaseaccesstype; + } + + public String getDatauploadtype() { + return datauploadtype; + } + + public void setDatauploadtype(final String datauploadtype) { + this.datauploadtype = datauploadtype; + } + + public String getDatabaseaccessrestriction() { + return databaseaccessrestriction; + } + + public void setDatabaseaccessrestriction(final String databaseaccessrestriction) { + this.databaseaccessrestriction = databaseaccessrestriction; + } + + public String getDatauploadrestriction() { + return datauploadrestriction; + } + + public void setDatauploadrestriction(final String datauploadrestriction) { + this.datauploadrestriction = datauploadrestriction; + } + + public Boolean getVersioning() { + return versioning; + } + + public void setVersioning(final Boolean versioning) { + this.versioning = versioning; + } + + public String getCitationguidelineurl() { + return citationguidelineurl; + } + + public void setCitationguidelineurl(final String citationguidelineurl) { + this.citationguidelineurl = citationguidelineurl; + } + + public String getQualitymanagementkind() { + return qualitymanagementkind; + } + + public void setQualitymanagementkind(final String qualitymanagementkind) { + this.qualitymanagementkind = qualitymanagementkind; + } + + public String getPidsystems() { + return pidsystems; + } + + public void setPidsystems(final String pidsystems) { + this.pidsystems = pidsystems; + } + + public String getCertificates() { + return certificates; + } + + public void setCertificates(final String certificates) { + this.certificates = certificates; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java new file mode 100644 index 00000000..74c8f7cb --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java @@ -0,0 +1,124 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource info model", description = "provides information about the datasource and its aggregation status") +public class DatasourceInfo { + + @ApiModelProperty(position = 0) + private long indexRecords; + + @ApiModelProperty(position = 1) + private long fundedContent; + + @ApiModelProperty(position = 2) + private long fulltexts; + + @ApiModelProperty(position = 3) + private String lastIndexingDate; + + @ApiModelProperty(position = 4) + private String firstHarvestDate; + + @ApiModelProperty(position = 5) + private DatasourceDetails datasource; + + @ApiModelProperty(position = 6) + private AggregationInfo lastCollection; + + @ApiModelProperty(position = 7) + private AggregationInfo lastTransformation; + + @ApiModelProperty(position = 8) + private List aggregationHistory; + + public DatasourceInfo() { + super(); + } + + public DatasourceInfo setIndexRecords(final long indexRecords) { + this.indexRecords = indexRecords; + return this; + } + + public DatasourceInfo setFundedContent(final long fundedContent) { + this.fundedContent = fundedContent; + return this; + } + + public DatasourceInfo setFulltexts(final long fulltexts) { + this.fulltexts = fulltexts; + return this; + } + + public DatasourceInfo setLastIndexingDate(final String lastIndexingDate) { + this.lastIndexingDate = lastIndexingDate; + return this; + } + + public DatasourceInfo setAggregationHistory(final List aggregationHistory) { + this.aggregationHistory = aggregationHistory; + return this; + } + + public DatasourceInfo setLastCollection(final AggregationInfo lastCollection) { + this.lastCollection = lastCollection; + return this; + } + + public DatasourceInfo setLastTransformation(final AggregationInfo lastTransformation) { + this.lastTransformation = lastTransformation; + return this; + } + + public long getIndexRecords() { + return indexRecords; + } + + public long getFundedContent() { + return fundedContent; + } + + public long getFulltexts() { + return fulltexts; + } + + public String getLastIndexingDate() { + return lastIndexingDate; + } + + public List getAggregationHistory() { + return aggregationHistory; + } + + public AggregationInfo getLastCollection() { + return lastCollection; + } + + public AggregationInfo getLastTransformation() { + return lastTransformation; + } + + public DatasourceDetails getDatasource() { + return datasource; + } + + public DatasourceInfo setDatasource(final DatasourceDetails datasource) { + this.datasource = datasource; + return this; + } + + public String getFirstHarvestDate() { + return firstHarvestDate; + } + + public void setFirstHarvestDate(final String firstHarvestDate) { + this.firstHarvestDate = firstHarvestDate; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java new file mode 100644 index 00000000..daae0a07 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceResponse.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonAutoDetect +public class DatasourceResponse extends Response { + + private List datasourceInfo = Lists.newArrayList(); + + public DatasourceResponse addDatasourceInfo(final T datasourceInfo) { + getDatasourceInfo().add(datasourceInfo); + return this; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public DatasourceResponse setDatasourceInfo(final List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java new file mode 100644 index 00000000..ecf9e77f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSearchResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceSearchResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceSearchResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java new file mode 100644 index 00000000..34fda827 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippet.java @@ -0,0 +1,49 @@ +package eu.dnetlib.openaire.dsm.domain; + +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides information about the datasource") +public class DatasourceSnippet { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java new file mode 100644 index 00000000..69dcab91 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetExtended.java @@ -0,0 +1,198 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.Date; +import java.util.Set; + +import javax.validation.constraints.Email; +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Datasource model", description = "provides extended information about the datasource") +public class DatasourceSnippetExtended { + + @NotBlank + @ApiModelProperty(position = 0) + private String id; + + @NotBlank + @ApiModelProperty(position = 2) + private String officialname; + + @NotBlank + @ApiModelProperty(position = 3) + private String englishname; + + @ApiModelProperty(position = 4) + private String websiteurl; + + @Email + @ApiModelProperty(position = 5) + private String registeredby; + + @ApiModelProperty(position = 6) + private Date registrationdate; + + @ApiModelProperty(position = 7) + private String eoscDatasourceType; + + @ApiModelProperty(position = 8) + private String logoUrl; + + @ApiModelProperty(position = 9) + private String description; + + @ApiModelProperty(position = 10) + private Boolean consentTermsOfUse; + + @ApiModelProperty(position = 11) + private Date consentTermsOfUseDate; + + @ApiModelProperty(position = 12) + private Date lastConsentTermsOfUseDate; + + @ApiModelProperty(position = 13) + private Boolean fullTextDownload; + + @ApiModelProperty(position = 14) + private Set organizations; + + @Deprecated + @ApiModelProperty(position = 15) + private String typology; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getRegisteredby() { + return registeredby; + } + + public void setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + } + + public String getLogoUrl() { + return logoUrl; + } + + public Boolean getConsentTermsOfUse() { + return consentTermsOfUse; + } + + public DatasourceSnippetExtended setConsentTermsOfUse(final Boolean consentTermsOfUse) { + this.consentTermsOfUse = consentTermsOfUse; + return this; + } + + public Date getConsentTermsOfUseDate() { + return consentTermsOfUseDate; + } + + public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + this.consentTermsOfUseDate = consentTermsOfUseDate; + return this; + } + + public Boolean getFullTextDownload() { + return fullTextDownload; + } + + public DatasourceSnippetExtended setFullTextDownload(final Boolean fullTextDownload) { + this.fullTextDownload = fullTextDownload; + return this; + } + + public DatasourceSnippetExtended setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + return this; + } + + public String getDescription() { + return description; + } + + public DatasourceSnippetExtended setDescription(final String description) { + this.description = description; + return this; + } + + public Set getOrganizations() { + return organizations; + } + + public DatasourceSnippetExtended setOrganizations(final Set organizations) { + this.organizations = organizations; + return this; + } + + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public DatasourceSnippetExtended setTypology(final String typology) { + this.typology = typology; + return this; + } + + public Date getLastConsentTermsOfUseDate() { + return lastConsentTermsOfUseDate; + } + + public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java new file mode 100644 index 00000000..a5636b65 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceSnippetResponse.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.List; + +@ApiModel +@JsonAutoDetect +public class DatasourceSnippetResponse extends Response { + + @ApiModelProperty(position = 1) + private List datasourceInfo; + + public DatasourceSnippetResponse(List datasourceInfo) { + super(); + this.datasourceInfo = datasourceInfo; + } + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java new file mode 100644 index 00000000..f6e3de96 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java @@ -0,0 +1,44 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; + +@JsonAutoDetect +@ApiModel(value = "Filter name", description = "List of the field names used to filter datasources") +public enum FilterName { + + id, + managed, + collectedfrom, // exact match + officialname, + englishname, + websiteurl, + contactemail, + registeredby, + eoscDatasourceType, + platform, // like match + country; // exact match on related organization + + public static FilterType type(final FilterName filterName) { + switch (filterName) { + case id: + case managed: + case collectedfrom: + return FilterType.exact; + case officialname: + case englishname: + case websiteurl: + case contactemail: + case registeredby: + case eoscDatasourceType: + case platform: + return FilterType.search; + case country: + return FilterType.searchOrgs; + default: + throw new IllegalStateException("unmapped filter type for: " + filterName); + } + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java new file mode 100644 index 00000000..5d15adfd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java @@ -0,0 +1,5 @@ +package eu.dnetlib.openaire.dsm.domain; + +public enum FilterType { + exact, search, searchOrgs +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java new file mode 100644 index 00000000..81dc2d67 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Header.java @@ -0,0 +1,115 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; +import java.util.Queue; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Lists; +import com.google.gson.GsonBuilder; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class Header { + + @ApiModelProperty(position = 0) + private long total; + + @ApiModelProperty(position = 1) + private int page; + + @ApiModelProperty(position = 2) + private int size; + + @ApiModelProperty(position = 3) + private long time; + + @ApiModelProperty(position = 4) + private int statusCode; + + @ApiModelProperty(position = 5) + private List errors = Lists.newArrayList(); + + @JsonIgnore + private Queue exceptions = Lists.newLinkedList(); + + public static Header newInsance() { + return new Header(); + } + + public Header() { + } + + public long getTime() { + return time; + } + + public Header setTime(final long time) { + this.time = time; + return this; + } + + public int getStatusCode() { + return statusCode; + } + + public Header setStatusCode(final int statusCode) { + this.statusCode = statusCode; + return this; + } + + public long getTotal() { + return total; + } + + public int getPage() { + return page; + } + + public int getSize() { + return size; + } + + public Header setPage(final int page) { + this.page = page; + return this; + } + + public Header setSize(final int size) { + this.size = size; + return this; + } + + public Header setTotal(final long total) { + this.total = total; + return this; + } + + public Queue getExceptions() { + return exceptions; + } + + public Header setExceptions(final Queue exceptions) { + this.exceptions = exceptions; + return this; + } + + public List getErrors() { + return getExceptions().stream() + .map(Throwable::getMessage) + .collect(Collectors.toList()); + } + + public Header setErrors(final List errors) { + this.errors = errors; + return this; + } + + public String toJson() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java new file mode 100644 index 00000000..ff389b38 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/IdentitiesDetails.java @@ -0,0 +1,32 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonAutoDetect +public class IdentitiesDetails { + + private String pid; + + private String issuertype; + + public String getPid() { + return pid; + } + + public String getIssuertype() { + return issuertype; + } + + public IdentitiesDetails setPid(final String pid) { + this.pid = pid; + return this; + } + + public IdentitiesDetails setIssuertype(final String issuertype) { + this.issuertype = issuertype; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java new file mode 100644 index 00000000..79dd5a3c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationDetails.java @@ -0,0 +1,76 @@ +package eu.dnetlib.openaire.dsm.domain; + +import javax.validation.constraints.NotBlank; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel(value = "Organization info model", description = "provides information about the organization") +public class OrganizationDetails extends OrganizationIgnoredProperties { + + @ApiModelProperty(position = 0) + private String legalshortname; + + @NotBlank + @ApiModelProperty(position = 1) + private String legalname; + + @ApiModelProperty(position = 2) + private String websiteurl; + + @ApiModelProperty(position = 3) + private String logourl; + + @NotBlank + @ApiModelProperty(position = 4) + private String country; + + public String getLegalshortname() { + return legalshortname; + } + + public String getLegalname() { + return legalname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public String getLogourl() { + return logourl; + } + + public String getCountry() { + return country; + } + + public OrganizationDetails setLegalshortname(final String legalshortname) { + this.legalshortname = legalshortname; + return this; + } + + public OrganizationDetails setLegalname(final String legalname) { + this.legalname = legalname; + return this; + } + + public OrganizationDetails setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + return this; + } + + public OrganizationDetails setLogourl(final String logourl) { + this.logourl = logourl; + return this; + } + + public OrganizationDetails setCountry(final String country) { + this.country = country; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java new file mode 100644 index 00000000..3212fb66 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/OrganizationIgnoredProperties.java @@ -0,0 +1,64 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.Date; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class OrganizationIgnoredProperties { + + @JsonIgnore + protected String id; + + @JsonIgnore + protected String collectedfrom; + + @JsonIgnore + protected Date dateofcollection; + + @JsonIgnore + protected String provenanceaction; + + @JsonIgnore + protected Set datasources; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCollectedfrom() { + return collectedfrom; + } + + public void setCollectedfrom(final String collectedfrom) { + this.collectedfrom = collectedfrom; + } + + public Date getDateofcollection() { + return dateofcollection; + } + + public void setDateofcollection(final Date dateofcollection) { + this.dateofcollection = dateofcollection; + } + + public String getProvenanceaction() { + return provenanceaction; + } + + public void setProvenanceaction(final String provenanceaction) { + this.provenanceaction = provenanceaction; + } + + public Set getDatasources() { + return datasources; + } + + public void setDatasources(final Set datasources) { + this.datasources = datasources; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java new file mode 100644 index 00000000..5bd17cc2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RegisteredDatasourceInfo.java @@ -0,0 +1,96 @@ +package eu.dnetlib.openaire.dsm.domain; + +public class RegisteredDatasourceInfo { + + private String id; + private String officialName; + private String englishName; + private String organization; + private String eoscDatasourceType; + private String registeredBy; + private String registrationDate; + private String compatibility; + private String lastCollectionDate; + private long lastCollectionTotal; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialName() { + return officialName; + } + + public void setOfficialName(final String officialName) { + this.officialName = officialName; + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public String getOrganization() { + return organization; + } + + public void setOrganization(final String organization) { + this.organization = organization; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + } + + public String getRegisteredBy() { + return registeredBy; + } + + public void setRegisteredBy(final String registeredBy) { + this.registeredBy = registeredBy; + } + + public String getRegistrationDate() { + return registrationDate; + } + + public void setRegistrationDate(final String registrationDate) { + this.registrationDate = registrationDate; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getLastCollectionDate() { + return lastCollectionDate; + } + + public void setLastCollectionDate(final String lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + } + + public long getLastCollectionTotal() { + return lastCollectionTotal; + } + + public void setLastCollectionTotal(final long lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java new file mode 100644 index 00000000..4275db07 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestFilter.java @@ -0,0 +1,20 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.HashMap; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; + +@JsonAutoDetect +@ApiModel(value = "Request filter", description = "field name and value pairs") +public class RequestFilter extends HashMap { + + /** + * + */ + private static final long serialVersionUID = 5501969842482508379L; + + public RequestFilter() {} + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java new file mode 100644 index 00000000..bf0c0cbd --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSort.java @@ -0,0 +1,8 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public enum RequestSort { + id, officialname, dateofvalidation, registrationdate +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java new file mode 100644 index 00000000..2423a991 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/RequestSortOrder.java @@ -0,0 +1,8 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public enum RequestSortOrder { + ASCENDING, DESCENDING +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java new file mode 100644 index 00000000..d52b499a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/Response.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@JsonAutoDetect +@ApiModel( + value = "Api response model", + description = "Api response model, provides a response header") +public class Response { + + @ApiModelProperty(position = 0) + private Header header; + + public Response() { + this.header = new Header(); + } + + public Header getHeader() { + return header; + } + + public Response setHeader(final Header header) { + this.header = header; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java new file mode 100644 index 00000000..cb79eb9c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleResponse.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +@JsonAutoDetect +public class SimpleResponse extends Response { + + @ApiModelProperty(position = 1) + private List response; + + public List getResponse() { + return response; + } + + public SimpleResponse setResponse(final List response) { + this.response = response; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java new file mode 100644 index 00000000..b4ccc38b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java @@ -0,0 +1,15 @@ +package eu.dnetlib.openaire.dsm.domain; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.AggregationInfo; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 29/11/2016. + */ +@ApiModel +@JsonAutoDetect +public class TransformationInfo extends AggregationInfo { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java new file mode 100755 index 00000000..04a7b7e9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiDbEntry.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import eu.dnetlib.enabling.datasources.common.Api; + +/** + * Api + */ +@Entity +@Table(name = "dsm_api") +public class ApiDbEntry extends Api { + + @Column(name = "compatibility_override") + protected String compatibilityOverride; + + public String getCompatibilityOverride() { + return compatibilityOverride; + } + + public Api setCompatibilityOverride(final String compatibilityOverride) { + this.compatibilityOverride = compatibilityOverride; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java new file mode 100644 index 00000000..59a90314 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamDbEntry.java @@ -0,0 +1,58 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.EmbeddedId; +import javax.persistence.Entity; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import eu.dnetlib.enabling.datasources.common.ApiParam; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 13/04/2017. + */ +@Entity +@Table(name = "dsm_apiparams") +@JsonIgnoreProperties(ignoreUnknown = true) +@ApiModel(value = "Datasource Api params model", description = "describes the datasource api params") +public class ApiParamDbEntry implements ApiParam { + + @EmbeddedId + protected ApiParamKeyDbEntry id; + + protected String value; + + public ApiParamDbEntry() {} + + public ApiParam setId(final ApiParamKeyDbEntry id) { + this.id = id; + return this; + } + + @JsonIgnore + public ApiParamKeyDbEntry getId() { + return id; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + @Override + public String getParam() { + return id.getParam(); + } + + @Override + public void setParam(final String param) { + final ApiParamKeyDbEntry apk = new ApiParamKeyDbEntry(); + apk.setParam(param); + setId(apk); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java new file mode 100644 index 00000000..7ffa6e08 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/ApiParamKeyDbEntry.java @@ -0,0 +1,28 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Embeddable; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import eu.dnetlib.enabling.datasources.common.ApiParamKey; + +/** + * Created by claudio on 13/04/2017. + */ +@Embeddable +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApiParamKeyDbEntry extends ApiParamKey { + + /** + * + */ + private static final long serialVersionUID = 1L; + + @Override + @JsonIgnore + public ApiDbEntry getApi() { + return super.getApi(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java new file mode 100644 index 00000000..87b78ed8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/CountryTerm.java @@ -0,0 +1,48 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.BrowseTerm; +import io.swagger.annotations.ApiModel; + +/** + * Created by claudio on 20/04/2017. + */ +@Entity +@Table(name = "browse_countries") +@ApiModel +@JsonAutoDetect +public class CountryTerm implements Comparable, BrowseTerm { + + @Id + private String term; + private long total; + + @Override + public String getTerm() { + return term; + } + + public void setTerm(final String term) { + this.term = term; + } + + @Override + public long getTotal() { + return total; + } + + public void setTotal(final long total) { + this.total = total; + } + + @Override + public int compareTo(final CountryTerm o) { + return getTerm().compareTo(o.getTerm()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java new file mode 100644 index 00000000..064c4f41 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceApiDbEntry.java @@ -0,0 +1,181 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import io.swagger.annotations.ApiModel; + +@Entity +@JsonAutoDetect +@Table(name = "dsm_datasource_api") +@ApiModel(value = "DatasourceApi model", description = "describes a joint view between datasources and their API (1:N)") +public class DatasourceApiDbEntry { + + @Id + @JsonIgnore + private Long rowid; + + private String id; + private String officialname; + private String englishname; + private String websiteurl; + private String contactemail; + private String collectedfrom; + private String registeredby; + @Column(name = "eosc_datasource_type") + private String eoscDatasourceType; + private String platform; + private Boolean managed; + + protected String protocol = null; + protected String contentdescription = null; + protected Boolean active = false; + protected Boolean removable = false; + + protected String compatibility; + private String baseurl; + + public DatasourceApiDbEntry() {} + + public Long getRowid() { + return rowid; + } + + public void setRowid(final Long rowid) { + this.rowid = rowid; + } + + public String getBaseurl() { + return baseurl; + } + + public void setBaseurl(final String baseurl) { + this.baseurl = baseurl; + } + + public String getContactemail() { + return contactemail; + } + + public void setContactemail(final String contactemail) { + this.contactemail = contactemail; + } + + public String getRegisteredby() { + return registeredby; + } + + public void setRegisteredby(final String registeredby) { + this.registeredby = registeredby; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getEnglishname() { + return englishname; + } + + public void setEnglishname(final String englishname) { + this.englishname = englishname; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + } + + public String getPlatform() { + return platform; + } + + public void setPlatform(final String platform) { + this.platform = platform; + } + + public Boolean getManaged() { + return managed; + } + + public void setManaged(final Boolean managed) { + this.managed = managed; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(final String protocol) { + this.protocol = protocol; + } + + public String getContentdescription() { + return contentdescription; + } + + public void setContentdescription(final String contentdescription) { + this.contentdescription = contentdescription; + } + + public Boolean getActive() { + return active; + } + + public void setActive(final Boolean active) { + this.active = active; + } + + public Boolean getRemovable() { + return removable; + } + + public void setRemovable(final Boolean removable) { + this.removable = removable; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCollectedfrom() { + return collectedfrom; + } + + public void setCollectedfrom(final String collectedfrom) { + this.collectedfrom = collectedfrom; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java new file mode 100755 index 00000000..fad3e190 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/DatasourceDbEntry.java @@ -0,0 +1,46 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import javax.persistence.Transient; + +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +import eu.dnetlib.enabling.datasources.common.Datasource; + +/** + * Datasource + */ +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_services") +public class DatasourceDbEntry extends Datasource { + + @Transient + private String openaireId; + + @Deprecated + @Column(name = "_typology_to_remove_") + private String typology; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public void setTypology(final String typology) { + this.typology = typology; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java new file mode 100644 index 00000000..032a4aa3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/IdentityDbEntry.java @@ -0,0 +1,20 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.Entity; +import javax.persistence.Table; + +import eu.dnetlib.enabling.datasources.common.Identity; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +/** + * Created by claudio on 13/04/2017. + */ + +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_identities") +public class IdentityDbEntry extends Identity { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java new file mode 100644 index 00000000..66e0fec8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/OrganizationDbEntry.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import java.util.Set; +import javax.persistence.Entity; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import eu.dnetlib.enabling.datasources.common.Organization; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +/** + * Created by claudio on 13/04/2017. + */ +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_organizations") +public class OrganizationDbEntry extends Organization { + + @Override + @JsonIgnore + public Set getDatasources() { + return super.getDatasources(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java new file mode 100644 index 00000000..42cee00f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemDbEntry.java @@ -0,0 +1,57 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import javax.persistence.EmbeddedId; +import javax.persistence.Entity; +import javax.persistence.Table; + +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SelectBeforeUpdate; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import eu.dnetlib.enabling.datasources.common.PidSystem; + +/** + * Created by claudio on 13/04/2017. + */ + +@Entity +@DynamicUpdate +@SelectBeforeUpdate +@Table(name = "dsm_pid_systems") +public class PidSystemDbEntry implements PidSystem { + + @EmbeddedId + protected PidSystemKeyDbEntry id; + + public PidSystem setId(final PidSystemKeyDbEntry id) { + this.id = id; + return this; + } + + @JsonIgnore + public PidSystemKeyDbEntry getId() { + return id; + } + + @Override + public String getType() { + return id.getType(); + } + + @Override + public void setType(final String type) { + id.setType(type); + } + + @Override + public String getScheme() { + return id.getScheme(); + } + + @Override + public void setScheme(final String scheme) { + id.setScheme(scheme); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java new file mode 100644 index 00000000..995d5685 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/db/PidSystemKeyDbEntry.java @@ -0,0 +1,62 @@ +package eu.dnetlib.openaire.dsm.domain.db; + +import java.io.Serializable; +import java.util.Objects; + +import javax.persistence.Embeddable; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import eu.dnetlib.enabling.datasources.common.PidSystem; + +@Embeddable +@JsonIgnoreProperties(ignoreUnknown = true) +public class PidSystemKeyDbEntry implements PidSystem, Serializable { + + private static final long serialVersionUID = 1L; + + private String service; + private String type; + private String scheme; + + public String getService() { + return service; + } + + public void setService(final String service) { + this.service = service; + } + + @Override + public String getType() { + return type; + } + + @Override + public void setType(final String type) { + this.type = type; + } + + @Override + public String getScheme() { + return scheme; + } + + @Override + public void setScheme(final String scheme) { + this.scheme = scheme; + } + + @Override + public int hashCode() { + return Objects.hash(scheme, service, type); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { return true; } + if (!(obj instanceof PidSystemKeyDbEntry)) { return false; } + final PidSystemKeyDbEntry other = (PidSystemKeyDbEntry) obj; + return Objects.equals(scheme, other.scheme) && Objects.equals(service, other.service) && Objects.equals(type, other.type); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java new file mode 100644 index 00000000..7c53b2c3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java @@ -0,0 +1,43 @@ +package eu.dnetlib.openaire.funders; + +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Component; + +import eu.dnetlib.openaire.funders.domain.ConversionUtils; +import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; +import eu.dnetlib.openaire.funders.domain.FunderDetails; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +public class FunderDao { + + @Autowired + private FunderRepository funderRepository; + + public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException { + return ConversionUtils + .asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId))); + } + + public List listFunderDetails(final int page, final int size) throws FundersApiException { + return funderRepository.findAll(PageRequest.of(page, size)) + .getContent() + .stream() + .map(ConversionUtils::asFunderDetails) + .collect(Collectors.toList()); + } + + public List listFunderIds(final int page, final int size) throws FundersApiException { + return funderRepository.findAll(PageRequest.of(page, size)) + .getContent() + .stream() + .map(f -> f.getId()) + .collect(Collectors.toList()); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java new file mode 100644 index 00000000..39bec085 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java @@ -0,0 +1,10 @@ +package eu.dnetlib.openaire.funders; + +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface FunderRepository extends JpaRepository { + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java new file mode 100644 index 00000000..6a3f72b8 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -0,0 +1,68 @@ +package eu.dnetlib.openaire.funders; + +import java.util.List; + +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.funders.domain.ExtendedFunderDetails; +import eu.dnetlib.openaire.funders.domain.FunderDetails; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.*; + +@RestController +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE funders API", description = "the OpenAIRE funders API") +public class FundersApiController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(FundersApiController.class); + + @Autowired + private FunderDao fDao; + + @RequestMapping(value = "/funders", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation( + value = "get basic information about funders", + notes = "basic information about funders: id, name, shortname, last update date, registration date", + response = FunderDetails[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = FunderDetails[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List getFunders( + @PathVariable final int page, + @PathVariable final int size) throws FundersApiException { + + return fDao.listFunderDetails(page, size); + } + + @RequestMapping(value = "/funder/{id}", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the funder details", notes = "complete funder information", response = FunderDbEntry.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = FunderDbEntry.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public ExtendedFunderDetails getFunderDetails( + @PathVariable final String id) throws FundersApiException { + + return fDao.getExtendedFunderDetails(id); + } + + @RequestMapping(value = "/funder/ids", produces = { "application/json" }, method = RequestMethod.GET) + @ApiOperation(value = "get the list of funder ids", notes = "get the list of funder ids", response = String[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public List getFunderIds( + @PathVariable final int page, + @PathVariable final int size + ) throws FundersApiException { + + return fDao.listFunderIds(page, size); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java new file mode 100644 index 00000000..c8f2f024 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiException.java @@ -0,0 +1,23 @@ +package eu.dnetlib.openaire.funders; + +public class FundersApiException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 842353818131133522L; + + public FundersApiException() {} + + public FundersApiException(final String message) { + super(message); + } + + public FundersApiException(final String message, final Throwable cause) { + super(message, cause); + } + + public FundersApiException(final Throwable cause) { + super(cause); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java new file mode 100644 index 00000000..75aae566 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ConversionUtils.java @@ -0,0 +1,88 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.io.StringReader; +import java.util.stream.Collectors; + +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.SAXReader; + +import static org.apache.commons.lang3.StringUtils.*; + +public class ConversionUtils { + + public static final String SEPARATOR = "::"; + + public static FunderDetails asFunderDetails(final FunderDbEntry fdb) { + final FunderDetails f = new FunderDetails(); + + f.setId(fdb.getId()); + f.setName(fdb.getName()); + f.setShortname(fdb.getShortname()); + f.setJurisdiction(fdb.getJurisdiction()); + f.setRegistrationDate(fdb.getRegistrationdate()); + f.setLastUpdateDate(fdb.getLastupdatedate()); + + return f; + } + + public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) { + final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb)); + + if (fdb.getFundingpaths() != null) { + f.setFundingStreams( + fdb.getFundingpaths().stream() + .map(ConversionUtils::asFundingStream) + .collect(Collectors.toList())); + } + + return f; + } + + private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) { + final FundingStream f = new FundingStream(); + + try { + final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath())); + + for(int i=2;i>=0;i--) { + if (hasFundingLevel(i, xml)) { + f.setId(getId(i, xml)); + f.setName(getName(i, xml)); + break; + } + } + + if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) { + f.setId(xml.valueOf("//funder/shortname/text()")); + f.setName(xml.valueOf("//funder/name/text()")); + } + + if (isBlank(f.getId())) { + throw new IllegalStateException("invalid funding path:\n" + xml.asXML()); + } + + return f; + + } catch (DocumentException e) { + throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath()); + } + } + + private static String getName(int level, final Document d) { + return d.valueOf(String.format("//funding_level_%s/description/text()", level)); + } + + private static String getId(int level, final Document d) { + return substringAfter( + d.valueOf( + String.format("//funding_level_%s/id/text()", level) + ), SEPARATOR); + } + + private static boolean hasFundingLevel(int level, Document d) { + return isNotBlank(getId(level, d)); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java new file mode 100644 index 00000000..d6cc54d2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/ExtendedFunderDetails.java @@ -0,0 +1,34 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class ExtendedFunderDetails extends FunderDetails { + + private List fundingStreams; + + public ExtendedFunderDetails() {} + + public ExtendedFunderDetails(final FunderDetails fd) { + super(); + setId(fd.getId()); + setName(fd.getName()); + setShortname(fd.getShortname()); + setJurisdiction(fd.getJurisdiction()); + setRegistrationDate(fd.getRegistrationDate()); + setLastUpdateDate(fd.getLastUpdateDate()); + } + + public List getFundingStreams() { + return fundingStreams; + } + + public ExtendedFunderDetails setFundingStreams(final List fundingStreams) { + this.fundingStreams = fundingStreams; + return this; + } + +} + diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java new file mode 100644 index 00000000..04335b09 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FunderDetails.java @@ -0,0 +1,76 @@ +package eu.dnetlib.openaire.funders.domain; + +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +@JsonAutoDetect +public class FunderDetails { + + private String id; + + private String name; + + private String shortname; + + private String jurisdiction; + + private Date registrationDate; + + private Date lastUpdateDate; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getShortname() { + return shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public Date getRegistrationDate() { + return registrationDate; + } + + public Date getLastUpdateDate() { + return lastUpdateDate; + } + + public FunderDetails setId(final String id) { + this.id = id; + return this; + } + + public FunderDetails setName(final String name) { + this.name = name; + return this; + } + + public FunderDetails setShortname(final String shortname) { + this.shortname = shortname; + return this; + } + + public FunderDetails setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + return this; + } + + public FunderDetails setRegistrationDate(final Date registrationDate) { + this.registrationDate = registrationDate; + return this; + } + + public FunderDetails setLastUpdateDate(final Date lastUpdateDate) { + this.lastUpdateDate = lastUpdateDate; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java new file mode 100644 index 00000000..1efd0aa7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/FundingStream.java @@ -0,0 +1,27 @@ +package eu.dnetlib.openaire.funders.domain; + +public class FundingStream { + + private String id; + + private String name; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public FundingStream setId(final String id) { + this.id = id; + return this; + } + + public FundingStream setName(final String name) { + this.name = name; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java new file mode 100644 index 00000000..a47c8d51 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -0,0 +1,116 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.sql.Date; +import java.util.Set; +import javax.persistence.*; + +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; + +@Entity +@Table(name = "funders") +public class FunderDbEntry { + + @Id + private String id; + private String name; + private String shortname; + private String jurisdiction; + private String websiteurl; + private String policy; + private Date registrationdate; + private Date lastupdatedate; + + @OneToMany( + cascade = { CascadeType.PERSIST, CascadeType.MERGE }, + fetch = FetchType.LAZY ) + @JoinTable( + name = "funder_identity", + joinColumns = @JoinColumn(name = "funder"), + inverseJoinColumns = @JoinColumn(name = "pid")) + private Set pids; + + @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private Set fundingpaths; + + public FunderDbEntry() {} + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortname() { + return shortname; + } + + public void setShortname(final String shortname) { + this.shortname = shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(final String policy) { + this.policy = policy; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public Date getLastupdatedate() { + return lastupdatedate; + } + + public void setLastupdatedate(final Date lastupdatedate) { + this.lastupdatedate = lastupdatedate; + } + + public Set getPids() { + return pids; + } + + public void setPids(final Set pids) { + this.pids = pids; + } + + public Set getFundingpaths() { + return fundingpaths; + } + + public void setFundingpaths(final Set fundingpaths) { + this.fundingpaths = fundingpaths; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java new file mode 100644 index 00000000..2f7dc534 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java @@ -0,0 +1,104 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import javax.persistence.*; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +@Entity +@Table(name = "fundingpaths") +public class FundingPathDbEntry { + + @JsonIgnore + @Transient + @Column(name = "_dnet_resource_identifier_") + private String dnetresourceidentifier; + + @Id + private String id; + private String path; + + @Column(name = "funder") + @Transient + @JsonIgnore + private String orgId; + private String jurisdiction; + private String description; + private String optional1; + private String optional2; + + private String funderid; + + public FundingPathDbEntry() {} + + public String getDnetresourceidentifier() { + return dnetresourceidentifier; + } + + public void setDnetresourceidentifier(final String dnetresourceidentifier) { + this.dnetresourceidentifier = dnetresourceidentifier; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getPath() { + return path; + } + + public void setPath(final String path) { + this.path = path; + } + + public String getOrgId() { + return orgId; + } + + public void setOrgId(final String orgId) { + this.orgId = orgId; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getOptional1() { + return optional1; + } + + public void setOptional1(final String optional1) { + this.optional1 = optional1; + } + + public String getOptional2() { + return optional2; + } + + public void setOptional2(final String optional2) { + this.optional2 = optional2; + } + + public String getFunderid() { + return funderid; + } + + public void setFunderid(final String funderid) { + this.funderid = funderid; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java new file mode 100644 index 00000000..a3027500 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/InfoController.java @@ -0,0 +1,103 @@ +package eu.dnetlib.openaire.info; + +import java.time.LocalDate; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.ExporterConstants; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; + +@RestController +@CrossOrigin(origins = { + "*" +}) +@ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE Info API", description = "the OpenAIRE info API") +public class InfoController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(InfoController.class); // NOPMD by marko on 11/24/08 5:02 PM + + public final static String UTF8 = "UTF-8"; + + @Autowired + private JdbcInfoDao jdbcInfoDao; + + @RequestMapping(value = "/info/{infoKey}", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get info date", notes = "get info date", tags = { + ExporterConstants.R + }, response = LocalDate.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public LocalDate getDate(@PathVariable final String infoKey) { + final JdbcInfoDao.DATE_INFO info = JdbcInfoDao.DATE_INFO.valueOf(infoKey); + if (info == null) { throw new RuntimeException(infoKey + " not recognized"); } + return jdbcInfoDao.getDate(info); + } + + @RequestMapping(value = "/info", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get all the info date", notes = "get all the info date", tags = { + ExporterConstants.R + }, response = Map.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public Map listInfo() { + final Map map = Maps.newHashMap(); + for (final JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()) { + map.put(dateInfo.name(), jdbcInfoDao.getDate(dateInfo)); + } + return map; + } + + @RequestMapping(value = "/info/keys", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "get the available keys", notes = "get the available keys", tags = { + ExporterConstants.R + }, response = String.class, responseContainer = "List") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = LocalDate.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) + }) + public List listInfoKeys() { + final List keys = Lists.newArrayList(); + for (final JdbcInfoDao.DATE_INFO dateInfo : JdbcInfoDao.DATE_INFO.values()) { + keys.add(dateInfo.name()); + } + return keys; + } + + @RequestMapping(value = "/info/dropCache", produces = { + "application/json" + }, method = RequestMethod.GET) + @ApiOperation(value = "Drops the info cache", notes = "Drops the info cache", tags = { + ExporterConstants.R + }) + public void dropCache() { + jdbcInfoDao.dropCache(); + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java new file mode 100644 index 00000000..0898d85e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDao.java @@ -0,0 +1,23 @@ +package eu.dnetlib.openaire.info; + +import java.time.LocalDate; + +public interface JdbcInfoDao { + + enum DATE_INFO { + claim_load_date, + oaf_load_date, + odf_load_date, + inference_date, + stats_update_date, + crossref_update_date, + orcid_update_date, + unpaywall_update_date, + mag_update_date + } + + LocalDate getDate(final DATE_INFO dateInfo); + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java new file mode 100644 index 00000000..344728d9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/info/JdbcInfoDaoImpl.java @@ -0,0 +1,87 @@ +package eu.dnetlib.openaire.info; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; + +import javax.sql.DataSource; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import eu.dnetlib.DnetOpenaireExporterProperties; + +/** + * Created by alessia on 29/04/2020 + * + * Get and set info dates via JDBC. Dates are expected to be in a table named 'info' with two columns: key - see JdbcInfoDao.DATE_INFO enum + * value - the date (LocalDate, no time) + * + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.info", havingValue = "true") +public class JdbcInfoDaoImpl implements JdbcInfoDao { + + private static final Log log = LogFactory.getLog(JdbcInfoDaoImpl.class); + + @Autowired + private DnetOpenaireExporterProperties config; + + @Autowired + private DataSource dataSource; + + @Override + @Cacheable("info") + public LocalDate getDate(final DATE_INFO dateInfo) { + final String sql = "SELECT value FROM info WHERE key=?"; + + LocalDate date = null; + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con, dateInfo.name()); final ResultSet rs = getRs(stm)) { + log.info("loading info " + dateInfo + " Query: " + stm.toString()); + if (rs.next()) { + date = rs.getObject("value", LocalDate.class); + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + + return date; + } + + @Override + @CacheEvict(cacheNames = { + "info" + }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped info cache"); + } + + private Connection getConn() throws SQLException { + final Connection connection = dataSource.getConnection(); + connection.setAutoCommit(false); + return connection; + } + + private PreparedStatement getStm(final String sql, final Connection con, final String param) throws SQLException { + final PreparedStatement stm = con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY); + stm.setString(1, param); + stm.setFetchSize(config.getJdbc().getMaxRows()); + return stm; + } + + private ResultSet getRs(final PreparedStatement stm) throws SQLException { + final ResultSet rs = stm.executeQuery(); + rs.setFetchSize(config.getJdbc().getMaxRows()); + return rs; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java new file mode 100644 index 00000000..376b0ade --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParams.java @@ -0,0 +1,82 @@ +package eu.dnetlib.openaire.project; + +import java.util.regex.Pattern; + +public class ProjectQueryParams { + + private final Pattern patternFundingStream = Pattern.compile("(\\w*(::|%| )*)*"); + + private final Pattern patternDate = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d"); + + private String fundingProgramme = null; + /** Whatever is following the fundingProgramme **/ + private String fundingPath = null; + + private String startFrom = null; + private String startUntil = null; + private String endFrom = null; + private String endUntil = null; + + public String getFundingProgramme() { + return fundingProgramme; + } + + public void setFundingProgramme(final String fundingProgramme) { + this.fundingProgramme = verifyParam(fundingProgramme); + } + + public String getFundingPath() { + return fundingPath; + } + + public void setFundingPath(final String fundingPath) { + this.fundingPath = verifyParam(fundingPath); + } + + public String getStartFrom() { + return startFrom; + } + + public void setStartFrom(final String startFrom) { + this.startFrom = verifyDateParam(startFrom); + } + + public String getStartUntil() { + return startUntil; + } + + public void setStartUntil(final String startUntil) { + this.startUntil = verifyDateParam(startUntil); + } + + public String getEndFrom() { + return endFrom; + } + + public void setEndFrom(final String endFrom) { + this.endFrom = verifyDateParam(endFrom); + } + + public String getEndUntil() { + return endUntil; + } + + public void setEndUntil(final String endUntil) { + this.endUntil = verifyDateParam(endUntil); + } + + protected String verifyParam(final String p) { + if (p != null && !patternFundingStream.matcher(p).matches()) { + throw new IllegalArgumentException(String.format("Parameter '%s' contains an invalid character", p)); + } + return p; + } + + protected String verifyDateParam(final String date) { + if (date != null && !patternDate.matcher(date).matches()) { + throw new IllegalArgumentException( + String.format("Parameter date '%s' contains an invalid character. Accepted pattern is %s", date, patternDate.toString())); + } + return date; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java new file mode 100644 index 00000000..8108b9db --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectQueryParamsFactory.java @@ -0,0 +1,47 @@ +package eu.dnetlib.openaire.project; + +import javax.servlet.http.HttpServletRequest; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public class ProjectQueryParamsFactory { + + private static final String BASE_PATH = "/export/"; + private static final String NO_FILTER = "ALL"; + + public ProjectQueryParams generateParams(final HttpServletRequest request, + final String startFrom, + final String startUntil, + final String endFrom, + final String endUntil) { + ProjectQueryParams params = new ProjectQueryParams(); + + String[] arr = request.getServletPath().replace(BASE_PATH, "").split("\\/"); + if (arr.length != 5) throw new IllegalArgumentException("Invalid url"); + + params.setFundingProgramme(arr[0]); + String stream = NO_FILTER.equals(arr[1]) ? null : arr[1]; + String substream = NO_FILTER.equals(arr[2]) ? null : arr[2]; + if (substream == null) { + params.setFundingPath(stream); + } else { + if (stream == null) { + stream = "%"; + } + params.setFundingPath(stream + "::" + substream); + } + // params.setSpecificProgramme(NO_FILTER.equals(arr[1]) ? null : arr[1]); + // params.setSubdivision(NO_FILTER.equals(arr[2]) ? null : arr[2]); + // NB: arr[3] should be 'projects' + // NB: arr[4] should be '[file].do' + params.setStartFrom(startFrom); + params.setStartUntil(startUntil); + params.setEndFrom(endFrom); + params.setEndUntil(endUntil); + + return params; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java new file mode 100644 index 00000000..00abbcf2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/ProjectsController.java @@ -0,0 +1,229 @@ +package eu.dnetlib.openaire.project; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.zip.ZipOutputStream; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.common.xml.XmlEscapers; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.DnetOpenaireExporterProperties.Project; +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.common.ExporterConstants; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import eu.dnetlib.openaire.project.dao.JdbcApiDao; +import eu.dnetlib.openaire.project.dao.ValueCleaner; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; + +@Controller +@CrossOrigin(origins = { "*" }) +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +@io.swagger.annotations.Api(tags = "OpenAIRE projects API", description = "the OpenAIRE projects API") +public class ProjectsController extends AbstractExporterController { + + private static final Log log = LogFactory.getLog(ProjectsController.class); // NOPMD by marko on 11/24/08 5:02 PM + + public final static String UTF8 = "UTF-8"; + + @Autowired + private DnetOpenaireExporterProperties config; + + @Autowired + private JdbcApiDao dao; + + @Autowired + private ProjectQueryParamsFactory projectQueryParamsFactory; + + @RequestMapping(value = "/export/**/project/dspace.do", method = RequestMethod.GET) + @ApiOperation( + value = "DSpace", + notes = "return project information in compatible with the OpenAIRE plugin for DSpace", + tags = { ExporterConstants.DSPACE }, + response = String.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processDspace(final HttpServletRequest request, final ServletResponse response, + @RequestParam(value = "startFrom", required = false) final String startFrom, + @RequestParam(value = "startUntil", required = false) final String startUntil, + @RequestParam(value = "endFrom", required = false) final String endFrom, + @RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { + + final Project conf = config.getProject(); + + final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); + final StringTemplate headSt = new StringTemplate(IOUtils.toString(conf.getDspaceHeadTemplate().getInputStream(), UTF8)); + + headSt.setAttribute("fundingProgramme", params.getFundingProgramme()); + + final StringTemplate tailSt = new StringTemplate(IOUtils.toString(conf.getDspaceTailTemplate().getInputStream(), UTF8)); + + response.setContentType("text/xml"); + doProcess(response, params, headSt.toString(), conf.getDspaceTemplate(), tailSt.toString(), s -> XmlEscapers.xmlContentEscaper().escape(oneLiner(s))); + } + + @RequestMapping(value = "/export/**/project/eprints.do", method = RequestMethod.GET) + @ApiOperation( + value = "EPrints", + notes = "return project information in compatible with the OpenAIRE plugin for Eprints", + tags = { ExporterConstants.EPRINT }, + response = String.class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = String.class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processEprints(final HttpServletRequest request, final ServletResponse response, + @RequestParam(value = "startFrom", required = false) final String startFrom, + @RequestParam(value = "startUntil", required = false) final String startUntil, + @RequestParam(value = "endFrom", required = false) final String endFrom, + @RequestParam(value = "endUntil", required = false) final String endUntil) throws Exception { + + final ProjectQueryParams params = projectQueryParamsFactory.generateParams(request, startFrom, startUntil, endFrom, endUntil); + response.setContentType("text/html"); + doProcess(response, params, null, config.getProject().getEprintsTemplate(), null, this::oneLiner); + } + + private String oneLiner(final String s) { + return StringUtils.isNotBlank(s) ? s.replaceAll("\\n", " ").trim() : ""; + } + + private void doProcess( + final ServletResponse response, + final ProjectQueryParams params, + final String head, final Resource projectTemplate, final String tail, + final ValueCleaner cleaner) throws IOException, SQLException { + + final StringTemplate st = new StringTemplate(IOUtils.toString(projectTemplate.getInputStream(), UTF8)); + try(final OutputStream out = new BufferedOutputStream(response.getOutputStream())) { + dao.streamProjects(obtainQuery(params), out, head, st, tail, cleaner); + } + } + + @RequestMapping(value = "/noads/project2tsv.do", method = RequestMethod.GET) + @ApiOperation( + value = "TSV", + notes = "download project information in TSV format", + tags = { ExporterConstants.TSV }, + response = ProjectTsv[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ProjectTsv[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void processTsv(final HttpServletResponse response, + @RequestParam(value = "funding", required = true) final String funding, + @RequestParam(value = "article293", required = false) final Boolean article293) throws Exception { + + final String fundingPrefix = getFundingPrefix(funding, null); + + final String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); + final String filename = "projects_" + funding + "_" + date + ".tsv"; + response.setContentType("text/tab-separated-values"); + response.setHeader("Content-Disposition", "attachment; filename=\"" + filename + ".zip\""); + try(final ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(response.getOutputStream()))) { + dao.processTsvRequest(out, article293, fundingPrefix, filename); + } catch (Throwable e) { + throw new RuntimeException("Error processing the request", e); + } + } + + @RequestMapping(value = "/export/streamProjectDetails.do", method = RequestMethod.GET) + @ApiOperation( + value = "Stream projects", + notes = "stream project information", + tags = { ExporterConstants.STREAMING }, + response = ProjectDetails[].class) + @ApiResponses(value = { + @ApiResponse(code = 200, message = "OK", response = ProjectDetails[].class), + @ApiResponse(code = 500, message = "unexpected error", response = ErrorMessage.class) }) + public void streamProjectDetails(final HttpServletResponse response, + @RequestParam(value = "format", required = true) final String format, + @RequestParam(value = "compress", required = false) final Boolean compress) throws IOException, SQLException { + + if (compress != null && compress) { + response.setHeader("Content-Encoding", "gzip"); + } + switch (format) { + case "csv": + response.setContentType("text/csv"); + break; + case "json": + response.setContentType("text/plain"); + break; + default: throw new IllegalArgumentException("unsupported format: " + format); + } + + dao.processProjectDetails(response.getOutputStream(), format, compress); + } + + /** + * Creates the query on the fundingProgramme specified in the given parameters. + * + * @param params + * request parameters + * @return the query string + * @throws IllegalArgumentException + * if the funding program is not recognized + * @throws IOException + * if there are problem loading the query temlate + * @throws IllegalArgumentException + * if the funding program is not recognized + */ + protected String obtainQuery(final ProjectQueryParams params) throws IllegalArgumentException, IOException { + String funding = params.getFundingProgramme(); + String suffix = params.getFundingPath(); + + final StringTemplate st = new StringTemplate(IOUtils.toString(config.getProject().getProjectsFundingQueryTemplate().getInputStream(), UTF8)); + st.setAttribute("fundingprefix", getFundingPrefix(funding, suffix)); + String theQuery = setDateParameters(st.toString(), params); + log.debug("Generated query: " + theQuery); + return theQuery; + } + + private String getFundingPrefix(final String funding, final String suffix) { + final Map fundingIds = dao.readFundingpathIds(); + if (!fundingIds.containsKey(funding.toUpperCase())) { + throw new IllegalArgumentException("invalid funding " + funding); + } + String fundingPrefix = fundingIds.get(funding.toUpperCase()); + return StringUtils.isBlank(suffix) ? fundingPrefix : fundingPrefix + "::" + suffix.toUpperCase(); + } + + private String setDateParameters(final String query, final ProjectQueryParams params) { + String queryWithDates = query; + if (params.getStartFrom() != null) { + queryWithDates += " AND startdate >= '" + params.getStartFrom() + "'"; + } + if (params.getStartUntil() != null) { + queryWithDates += " AND startdate <= '" + params.getStartUntil() + "'"; + } + if (params.getEndFrom() != null) { + queryWithDates += " AND enddate >= '" + params.getEndFrom() + "'"; + } + if (params.getEndUntil() != null) { + queryWithDates += " AND enddate <= '" + params.getEndUntil() + "'"; + } + return queryWithDates; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java new file mode 100644 index 00000000..277c5c29 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDao.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.project.dao; + +import java.io.IOException; +import java.io.OutputStream; +import java.sql.SQLException; +import java.util.Map; +import java.util.zip.ZipOutputStream; + +import org.antlr.stringtemplate.StringTemplate; + +public interface JdbcApiDao { + + Map readFundingpathIds(); + + void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException; + + void processTsvRequest(final ZipOutputStream out, final Boolean article293, final String fundingPrefix, final String filename) throws IOException; + + void streamProjects( + final String sql, + final OutputStream out, + final String head, + final StringTemplate projectTemplate, + final String tail, + final ValueCleaner cleaner) throws IOException, SQLException; + + void dropCache(); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java new file mode 100644 index 00000000..49a0279f --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/JdbcApiDaoImpl.java @@ -0,0 +1,238 @@ +package eu.dnetlib.openaire.project.dao; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.sql.*; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import javax.sql.DataSource; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import eu.dnetlib.DnetOpenaireExporterProperties; +import eu.dnetlib.openaire.project.domain.Project; +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Created by claudio on 20/09/16. + */ +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public class JdbcApiDaoImpl implements JdbcApiDao { + + public static final Charset UTF8 = Charset.forName("UTF-8"); + + private static final Log log = LogFactory.getLog(JdbcApiDaoImpl.class); + + @Autowired + private DnetOpenaireExporterProperties config; + + @Autowired + private DataSource dataSource; + + @Autowired + private ProjectTsvRepository projectTsvRepository; + + @Override + @Cacheable("fundingpath-ids") + public Map readFundingpathIds() { + + log.debug("loading funding ids"); + final String sql = "SELECT id FROM fundingpaths"; + final Set ids = Sets.newHashSet(); + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + ids.add(rs.getString("id")); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + log.debug(String.format("loaded %s funding ids", ids.size())); + + final Map res = Maps.newHashMap(); + final Splitter sp = Splitter.on("::").trimResults(); + ids.stream() + .filter(s -> sp.splitToList(s).size() < 3) + .forEach(s -> res.put(StringUtils.substringAfterLast(s, "::").toUpperCase(), s)); + + res.put("FP7", "ec__________::EC::FP7"); + res.put("H2020", "ec__________::EC::H2020"); + log.debug(String.format("processed %s funding ids", res.size())); + res.forEach((k,v) -> log.debug(String.format("%s : '%s'", k, v))); + return res; + } + + @Override + public void processProjectDetails(final OutputStream outputStream, String format, Boolean compress) throws IOException { + final OutputStream out = getOutputStream(new BufferedOutputStream(outputStream), compress); + try { + final String sql = "SELECT * FROM project_details"; + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + try { + switch (format) { + case "csv": + out.write(getProjectDetails(rs).asCSV().getBytes(UTF8)); + break; + case "json": + out.write(getProjectDetails(rs).asJson().getBytes(UTF8)); + break; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } finally { + if (out instanceof GZIPOutputStream) { + ((GZIPOutputStream) out).finish(); + } + out.close(); + } + } + + private OutputStream getOutputStream(final OutputStream outputStream, final Boolean compress) throws IOException { + if (compress != null && compress) { + return new GZIPOutputStream(outputStream); + } + return outputStream; + } + + private ProjectDetails getProjectDetails(final ResultSet rs) throws SQLException { + return new ProjectDetails() + .setProjectId(rs.getString("projectid")) + .setAcronym(rs.getString("acronym")) + .setCode(rs.getString("code")) + .setJsonextrainfo(rs.getString("jsonextrainfo")) + .setFundingPath(asList(rs.getArray("fundingpath"))); + } + + private String[] asList(final Array value) throws SQLException { + if (value != null) { + final List list = Arrays.asList((Object[]) value.getArray()); + return list.stream() + .map(o -> o != null ? o.toString() : null) + .toArray(String[]::new); + } + return new String[0]; + } + + @Override + public void processTsvRequest(final ZipOutputStream out, final Boolean article293, final String fundingPrefix, final String filename) throws IOException { + out.putNextEntry(new ZipEntry(filename)); + writeTsvLine(out, Splitter.on(",").trimResults().splitToList(config.getProject().getTsvFields())); + queryForTsv(fundingPrefix, article293).forEach(p -> { + try { + writeTsvLine(out, p.asList()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void writeTsvLine(final ZipOutputStream out, final List s) throws IOException { + out.write(Joiner.on('\t').useForNull("").join(s).getBytes(UTF8)); + out.write('\n'); + } + + private Iterable queryForTsv(final String fundingPrefix, final Boolean article293) { + log.debug(String.format("fundingPrefix:'%s' and oa_mandate_for_datasets:'%s'", fundingPrefix, article293)); + if (article293 != null) { + return projectTsvRepository.findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(fundingPrefix, article293); + } else { + return projectTsvRepository.findByFundingpathidStartingWithOrderByAcronym(fundingPrefix); + } + } + + @Override + public void streamProjects(final String sql, final OutputStream out, + final String head, final StringTemplate projectTemplate, final String tail, + final ValueCleaner cleaner) throws IOException, SQLException { + + if (log.isDebugEnabled()) { + log.debug("Thread " + Thread.currentThread().getId() + " begin"); + } + final LocalDateTime start = LocalDateTime.now(); + + if (StringUtils.isNotBlank(head)) { + out.write(head.getBytes(UTF8)); + } + + try (final Connection con = getConn(); final PreparedStatement stm = getStm(sql, con); final ResultSet rs = getRs(stm)) { + while (rs.next()) { + final Project p = new Project() + .setFunder(cleaner.clean(rs.getString("funder"))) + .setJurisdiction(cleaner.clean(rs.getString("jurisdiction"))) + .setFundingpathid(cleaner.clean(rs.getString("fundingpathid"))) + .setAcronym(cleaner.clean(rs.getString("acronym"))) + .setTitle(cleaner.clean(rs.getString("title"))) + .setCode(cleaner.clean(rs.getString("code"))) + .setStartdate(cleaner.clean(rs.getString("startdate"))) + .setEnddate(cleaner.clean(rs.getString("enddate"))); + + projectTemplate.reset(); + projectTemplate.setAttribute("p", p); + out.write(projectTemplate.toString().getBytes(UTF8)); + } + if (StringUtils.isNotBlank(tail)) { + out.write(tail.getBytes(UTF8)); + } + final LocalDateTime end = LocalDateTime.now(); + if (log.isDebugEnabled()) { + log.debug("Thread " + Thread.currentThread().getId() + " ends, took: " + Duration.between(start, end).toMillis() + " ms"); + } + } + } + + @Override + @CacheEvict(cacheNames = { "fundingpath-ids" }, allEntries = true) + @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") + public void dropCache() { + log.debug("dropped fundingpath ids cache"); + } + + private Connection getConn() throws SQLException { + final Connection connection = dataSource.getConnection(); + connection.setAutoCommit(false); + return connection; + } + + private PreparedStatement getStm(final String sql, final Connection con) throws SQLException { + final PreparedStatement stm = con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY); + stm.setFetchSize(config.getJdbc().getMaxRows()); + + return stm; + } + + private ResultSet getRs(final PreparedStatement stm) throws SQLException { + final ResultSet rs = stm.executeQuery(); + rs.setFetchSize(config.getJdbc().getMaxRows()); + return rs; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java new file mode 100644 index 00000000..a3e6f733 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectApiRepository.java @@ -0,0 +1,18 @@ +package eu.dnetlib.openaire.project.dao; + +import eu.dnetlib.openaire.project.domain.db.ProjectApi; +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 06/07/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public interface ProjectApiRepository extends PagingAndSortingRepository { + + Iterable findByFundingpathidStartingWith(String fundingpathid); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java new file mode 100644 index 00000000..f8328735 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ProjectTsvRepository.java @@ -0,0 +1,19 @@ +package eu.dnetlib.openaire.project.dao; + +import eu.dnetlib.openaire.project.domain.db.ProjectTsv; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by claudio on 04/07/2017. + */ +@Repository +@ConditionalOnProperty(value = "openaire.exporter.enable.project", havingValue = "true") +public interface ProjectTsvRepository extends PagingAndSortingRepository { + + Iterable findByFundingpathidStartingWithOrderByAcronym(String fundingpathid); + + Iterable findByFundingpathidStartingWithAndOaMandateForDatasetsOrderByAcronym(String fundingpathid, boolean article293); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java new file mode 100644 index 00000000..db84257c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/dao/ValueCleaner.java @@ -0,0 +1,10 @@ +package eu.dnetlib.openaire.project.dao; + +/** + * Created by claudio on 23/09/2016. + */ +public interface ValueCleaner { + + String clean(String s); + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java new file mode 100644 index 00000000..509351ff --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/Project.java @@ -0,0 +1,264 @@ +package eu.dnetlib.openaire.project.domain; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 20/09/16. + */ +public class Project { + + public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/"; + private String code; + private String acronym; + private String title; + private String callIdentifier; + private String startdate; + private String enddate; + private boolean oaMandateForPublications; + private boolean oaMandateForDatasets; + private String fundingpathid; + private String description; + private String funder; + private String jurisdiction; + private String orgLegalname; + private String orgCountry; + private String orgRole; + private String firstname; + private String secondnames; + private String email; + + public Project() { + } + + public String getIdnamespace() { + String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/"; + final String fundingProgram = asFundingProgram(getFundingpathid()); + if (StringUtils.isNotBlank(fundingProgram)) { + res += fundingProgram; + } + res += "/" + escapeCode(getCode()); + if (StringUtils.isNotBlank(getJurisdiction())) { + res += "/" + getJurisdiction(); + } + return res; + } + + public String getListLabel() { + return String.format("for:value:component:_%s_project_id", asFunder(getFunder())); + } + + private String asFunder(final String legalshortname) { + switch (legalshortname.toLowerCase()) { + case "ec": + return asFundingProgram(getFundingpathid()).toLowerCase(); + default: + return legalshortname.toLowerCase(); + } + } + + public List asList() { + return Lists.newArrayList( + clean(getCode()), + clean(getAcronym()), + clean(getTitle()), + clean(getCallIdentifier()), + clean(getStartdate()), + clean(getEnddate()), + String.valueOf(isOaMandateForPublications()), + String.valueOf(isOaMandateForDatasets()), + clean(getDescription()), + clean(getOrgLegalname()), + clean(getOrgCountry()), + clean(getOrgRole()), + clean(getFirstname()), + clean(getSecondnames()), + clean(getEmail())); + } + + private String clean(final String s) { + return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : ""; + } + + private String escapeCode(final String code) { + return replaceSlash(code); + } + + private String asFundingProgram(final String fundingpathid) { + final ArrayList strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid)); + if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid); + if(strings.size() == 2) return ""; + else return replaceSlash(strings.get(2)); + } + + private String replaceSlash(final String s) { + return s.replaceAll("/", "%2F"); + } + + public String getCode() { + return code; + } + + public Project setCode(final String code) { + this.code = code; + return this; + } + + public String getAcronym() { + return acronym; + } + + public Project setAcronym(final String acronym) { + this.acronym = acronym; + return this; + } + + public String getTitle() { + return title; + } + + public Project setTitle(final String title) { + this.title = title; + return this; + } + + public String getCallIdentifier() { + return callIdentifier; + } + + public Project setCallIdentifier(final String call_identifier) { + this.callIdentifier = call_identifier; + return this; + } + + public String getStartdate() { + return startdate; + } + + public Project setStartdate(final String startdate) { + this.startdate = startdate; + return this; + } + + public String getEnddate() { + return enddate; + } + + public Project setEnddate(final String enddate) { + this.enddate = enddate; + return this; + } + + public boolean isOaMandateForPublications() { + return oaMandateForPublications; + } + + public Project setOaMandateForPublications(final boolean oaMandateForPublications) { + this.oaMandateForPublications = oaMandateForPublications; + return this; + } + + public boolean isOaMandateForDatasets() { + return oaMandateForDatasets; + } + + public Project setOaMandateForDatasets(final boolean oaMandateForDatasets) { + this.oaMandateForDatasets = oaMandateForDatasets; + return this; + } + + + public String getFundingpathid() { + return fundingpathid; + } + + public Project setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + return this; + } + + public String getDescription() { + return description; + } + + public Project setDescription(final String description) { + this.description = description; + return this; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public Project setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + return this; + } + + public String getOrgLegalname() { + return orgLegalname; + } + + public Project setOrgLegalname(final String legalname) { + this.orgLegalname = legalname; + return this; + } + + public String getOrgCountry() { + return orgCountry; + } + + public Project setOrgCountry(final String country) { + this.orgCountry = country; + return this; + } + + public String getOrgRole() { + return orgRole; + } + + public Project setOrgRole(final String role) { + this.orgRole = role; + return this; + } + + public String getFirstname() { + return firstname; + } + + public Project setFirstname(final String firstname) { + this.firstname = firstname; + return this; + } + + public String getSecondnames() { + return secondnames; + } + + public Project setSecondnames(final String secondnames) { + this.secondnames = secondnames; + return this; + } + + public String getEmail() { + return email; + } + + public Project setEmail(final String email) { + this.email = email; + return this; + } + + public String getFunder() { + return funder; + } + + public Project setFunder(final String funder) { + this.funder = funder; + return this; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java new file mode 100644 index 00000000..361f6534 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectApi.java @@ -0,0 +1,152 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.sql.Date; +import java.util.ArrayList; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 20/09/16. + */ +@Entity +@Table(name = "projects_api") +@ApiModel(value = "Project api model", description = "Project api model used by DSpace and Eprints exporter") +public class ProjectApi { + + public static final String INFO_EU_REPO_GRANT_AGREEMENT = "info:eu-repo/grantAgreement/"; + + @Id + @JsonIgnore + private String id; + + private String code; + private String acronym; + private String title; + private String funder; + private String jurisdiction; + private Date startdate; + private Date enddate; + private String fundingpathid; + + public ProjectApi() { } + + public String getIdnamespace() { + String res = INFO_EU_REPO_GRANT_AGREEMENT + getFunder()+"/"; + final String fundingProgram = asFundingProgram(getFundingpathid()); + if (StringUtils.isNotBlank(fundingProgram)) { + res += fundingProgram; + } + res += "/" + escapeCode(getCode()); + if (StringUtils.isNotBlank(getJurisdiction())) { + res += "/" + getJurisdiction(); + } + return res; + } + + public String getListLabel() { + return String.format("for:value:component:_%s_project_id", asFunder(getFunder())); + } + + private String asFunder(final String legalshortname) { + switch (legalshortname.toLowerCase()) { + case "ec": + return asFundingProgram(getFundingpathid()).toLowerCase(); + default: + return legalshortname.toLowerCase(); + } + } + + private String escapeCode(final String code) { + return replaceSlash(code); + } + + private String asFundingProgram(final String fundingpathid) { + final ArrayList strings = Lists.newArrayList(Splitter.on("::").split(fundingpathid)); + if(strings.size() <= 1) throw new IllegalStateException("Unexpected funding id: "+fundingpathid); + if(strings.size() == 2) return ""; + else return replaceSlash(strings.get(2)); + } + + private String replaceSlash(final String s) { + return s.replaceAll("/", "%2F"); + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public String getFunder() { + return funder; + } + + public void setFunder(final String funder) { + this.funder = funder; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public Date getStartdate() { + return startdate; + } + + public void setStartdate(final Date startdate) { + this.startdate = startdate; + } + + public Date getEnddate() { + return enddate; + } + + public void setEnddate(final Date enddate) { + this.enddate = enddate; + } + + public String getFundingpathid() { + return fundingpathid; + } + + public void setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java new file mode 100644 index 00000000..e4bb8ad3 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectDetails.java @@ -0,0 +1,157 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.List; +import javax.persistence.*; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.gson.Gson; +import io.swagger.annotations.ApiModel; +import org.hibernate.annotations.Type; +import org.supercsv.cellprocessor.Optional; +import org.supercsv.cellprocessor.ift.CellProcessor; +import org.supercsv.cellprocessor.ift.StringCellProcessor; +import org.supercsv.io.CsvBeanReader; +import org.supercsv.io.CsvBeanWriter; +import org.supercsv.io.ICsvBeanReader; +import org.supercsv.io.ICsvBeanWriter; +import org.supercsv.prefs.CsvPreference; +import org.supercsv.util.CsvContext; + +/** + * Created by claudio on 04/07/2017. + */ +@Entity +@Table(name = "project_details") +@ApiModel(value = "Project details model", description = "provides project details") +public class ProjectDetails { + + @Transient + @JsonIgnore + private transient static final String[] FIELDS = Arrays.stream(ProjectDetails.class.getDeclaredFields()) + .map(Field::getName) + .filter(s -> !s.equals("FIELDS")) + .filter(s -> !s.startsWith("optional")) + .toArray(String[]::new); + + @Id + @Column(name = "projectid") + private String projectId; + private String acronym; + private String code; + + private String jsonextrainfo; + + @Type(type = "eu.dnetlib.openaire.common.GenericArrayUserType") + @Column(name = "fundingpath", columnDefinition = "text[]") + private String[] fundingPath; + + public ProjectDetails() { + } + + public String getProjectId() { + return projectId; + } + + public String getAcronym() { + return acronym; + } + + public String getCode() { + return code; + } + + public String getJsonextrainfo() { + return jsonextrainfo; + } + + public String[] getFundingPath() { + return fundingPath; + } + + public void setFundingPath(final List fundingPath) { + if(fundingPath != null && !fundingPath.isEmpty()) { + this.fundingPath = fundingPath.toArray(new String[fundingPath.size()]); + } + } + + public static ProjectDetails fromJson(final String json) { + return new Gson().fromJson(json, ProjectDetails.class); + } + + public static ProjectDetails fromCSV(final String csv) throws IOException { + try (ICsvBeanReader beanReader = new CsvBeanReader(new StringReader(csv), CsvPreference.STANDARD_PREFERENCE)) { + return beanReader.read(ProjectDetails.class, FIELDS, getProcessors(new StringCellProcessor() { + @Override + public Object execute(final Object value, final CsvContext context) { + return new Gson().fromJson(value.toString(), List.class); + } + })); + } + } + + /** + * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. Empty + * columns are read as null (hence the NotNull() for mandatory columns). + * + * @return the cell processors + */ + private static CellProcessor[] getProcessors(final CellProcessor fundingPathProcessor) { + return new CellProcessor[] { + new Optional(), // projectId + new Optional(), // acronym + new Optional(), // code + new Optional(), // jsonextrainfo + fundingPathProcessor + }; + } + + public String asJson() { + return new Gson().toJson(this) + '\n'; + } + + public String asCSV() throws IOException { + final StringWriter sb = new StringWriter(); + try (ICsvBeanWriter beanWriter = new CsvBeanWriter(sb, CsvPreference.STANDARD_PREFERENCE)) { + beanWriter.write(this, FIELDS, getProcessors(new StringCellProcessor() { + @Override + public Object execute(final Object value, final CsvContext context) { + return new Gson().toJson(value); + } + })); + beanWriter.flush(); + } + + return sb.toString(); + } + + public ProjectDetails setProjectId(final String projectId) { + this.projectId = projectId; + return this; + } + + public ProjectDetails setAcronym(final String acronym) { + this.acronym = acronym; + return this; + } + + public ProjectDetails setCode(final String code) { + this.code = code; + return this; + } + + public ProjectDetails setJsonextrainfo(final String jsonextrainfo) { + this.jsonextrainfo = jsonextrainfo; + return this; + } + + public ProjectDetails setFundingPath(final String[] fundingPath) { + this.fundingPath = fundingPath; + return this; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java new file mode 100644 index 00000000..fdf7bd05 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/project/domain/db/ProjectTsv.java @@ -0,0 +1,210 @@ +package eu.dnetlib.openaire.project.domain.db; + +import java.sql.Date; +import java.util.List; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Lists; +import io.swagger.annotations.ApiModel; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by claudio on 05/07/2017. + */ +@Entity +@Table(name = "projects_tsv") +@ApiModel(value = "Project TSV model", description = "project TSV model description") +public class ProjectTsv { + + @Id + @JsonIgnore + private long rowid; + private String code; + private String acronym; + private String title; + @Column(name = "call_identifier") + private String callIdentifier; + private Date startdate; + private Date enddate; + @Column(name = "ec_sc39") + private Boolean ecSc39; + @Column(name = "oa_mandate_for_publications") + private Boolean oaMandateForPublications; + @Column(name = "oa_mandate_for_datasets") + private Boolean oaMandateForDatasets; + @JsonIgnore + private String fundingpathid; + private String description; + @Column(name = "legalname") + private String orgLegalname; + @Column(name = "country") + private String orgCountry; + @Column(name = "role") + private String orgRole; + private String contactfullname; + private String contactemail; + + public ProjectTsv() {} + + public List asList() { + return Lists.newArrayList( + clean(getCode()), + clean(getAcronym()), + clean(getTitle()), + clean(getCallIdentifier()), + clean(getStartdate() != null ? getStartdate().toString() : ""), + clean(getEnddate() != null ? getEnddate().toString() : ""), + clean(String.valueOf(isOaMandateForPublications())), + clean(String.valueOf(isOaMandateForDatasets())), + clean(getDescription()), + clean(getOrgLegalname()), + clean(getOrgCountry()), + clean(getOrgRole()), + clean(getContactfullname()), + clean(getContactemail())); + } + + private String clean(final String s) { + return StringUtils.isNotBlank(s) ? "\"" + s.replaceAll("\\n|\\t|\\s+", " ").replace("\"","\"\"").trim() + "\"" : ""; + } + + public long getRowid() { + return rowid; + } + + public void setRowid(final long rowid) { + this.rowid = rowid; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(final String acronym) { + this.acronym = acronym; + } + + public String getTitle() { + return title; + } + + public void setTitle(final String title) { + this.title = title; + } + + public String getCallIdentifier() { + return callIdentifier; + } + + public void setCallIdentifier(final String callIdentifier) { + this.callIdentifier = callIdentifier; + } + + public Date getStartdate() { + return startdate; + } + + public void setStartdate(final Date startdate) { + this.startdate = startdate; + } + + public Date getEnddate() { + return enddate; + } + + public void setEnddate(final Date enddate) { + this.enddate = enddate; + } + + public Boolean isEcSc39() { + return ecSc39; + } + + public void setEcSc39(final Boolean ecSc39) { + this.ecSc39 = ecSc39; + } + + public Boolean isOaMandateForPublications() { + return oaMandateForPublications; + } + + public void setOaMandateForPublications(final Boolean oaMandateForPublications) { + this.oaMandateForPublications = oaMandateForPublications; + } + + public Boolean isOaMandateForDatasets() { + return oaMandateForDatasets; + } + + public void setOaMandateForDatasets(final Boolean oaMandateForDatasets) { + this.oaMandateForDatasets = oaMandateForDatasets; + } + + public String getFundingpathid() { + return fundingpathid; + } + + public void setFundingpathid(final String fundingpathid) { + this.fundingpathid = fundingpathid; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getOrgLegalname() { + return orgLegalname; + } + + public void setOrgLegalname(final String orgLegalname) { + this.orgLegalname = orgLegalname; + } + + public String getOrgCountry() { + return orgCountry; + } + + public void setOrgCountry(final String orgCountry) { + this.orgCountry = orgCountry; + } + + public String getOrgRole() { + return orgRole; + } + + public void setOrgRole(final String orgRole) { + this.orgRole = orgRole; + } + + public String getContactfullname() { + return contactfullname; + } + + public void setContactfullname(final String contactfullname) { + this.contactfullname = contactfullname; + } + + public String getContactemail() { + return contactemail; + } + + public void setContactemail(final String contactemail) { + this.contactemail = contactemail; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java new file mode 100644 index 00000000..73c54763 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Country.java @@ -0,0 +1,35 @@ +package eu.dnetlib.openaire.vocabularies; + +/** + * Created by claudio on 15/09/2017. + */ +public class Country { + + private String code; + + private String name; + + public Country() { + } + + public Country(final String code, final String name) { + this.code = code; + this.name = name; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java new file mode 100644 index 00000000..30adc410 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Term.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.vocabularies; + +/** + * Created by claudio on 15/09/2017. + */ +public class Term { + private String englishName; + private String code; + + public Term() { + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public String getCode() { + return code; + } + + public void setCode(final String code) { + this.code = code; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java new file mode 100644 index 00000000..54c6e090 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/vocabularies/Vocabulary.java @@ -0,0 +1,50 @@ +package eu.dnetlib.openaire.vocabularies; + +import java.util.List; +import java.util.Map; + +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Created by claudio on 15/09/2017. + */ +public class Vocabulary { + + + private static final Log log = LogFactory.getLog(Vocabulary.class); + + private List terms; + + public Vocabulary() { + } + + private static final Map byCode = Maps.newConcurrentMap(); + + public String getEnglishName(final String code) { + if (byCode.isEmpty()) { + hashByCode(); + } + final Term term = byCode.get(code); + return term != null ? term.getEnglishName() : null; + } + + public boolean hasCode(final String code) { + return getEnglishName(code) != null; + } + + private void hashByCode() { + log.info("hashing vocabulary by code ..."); + getTerms().forEach(term -> byCode.put(term.getCode(), term)); + log.info("hashing vocabulary by code ... done!"); + } + + public List getTerms() { + return terms; + } + + public void setTerms(final List terms) { + this.terms = terms; + } +} diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties new file mode 100644 index 00000000..2b947038 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -0,0 +1,35 @@ +# COMMON +server.servlet.context-path = /openaire +server.port = 8080 + +spring.datasource.driverClassName = org.postgresql.Driver +spring.jpa.database-platform = org.hibernate.dialect.PostgreSQL9Dialect +spring.jpa.show-sql = false +spring.jpa.properties.hibernate.format_sql = true +spring.jpa.hibernate.ddl-auto = validate +#spring.mvc.dispatch-options-request = true + +# SWAGGER +spring.jackson.date-format = eu.dnetlib.openaire.common.RFC3339DateFormat +spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS = false + +management.endpoints.web.exposure.include = prometheus,health + +management.endpoints.web.base-path = / +management.endpoints.web.path-mapping.prometheus = metrics +management.endpoints.web.path-mapping.health = health + + +# ENABLE / DISABLE CONTROLLERS +openaire.exporter.enable.dsm = true +openaire.exporter.enable.community = true +openaire.exporter.enable.context = true +openaire.exporter.enable.funders = false +openaire.exporter.enable.project = true +openaire.exporter.enable.info = true + +# CACHE TTL, 12h +openaire.exporter.cache.ttl = 43200000 + +maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st new file mode 100644 index 00000000..b75fcc9c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/projects_fundings.sql.st @@ -0,0 +1,11 @@ +SELECT + funder, + jurisdiction, + fundingpathid, + acronym, + title, + code, + startdate, + enddate +FROM projects_api +WHERE fundingpathid like '$fundingprefix$%' diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st new file mode 100644 index 00000000..56bff2be --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st @@ -0,0 +1,25 @@ +select + d.id as id, + d.officialname as "officialName", + d.englishname as "englishName", + o.legalname as organization, + d.eosc_datasource_type as eosc_datasource_type, + d.registeredby as "registeredBy", + d.registrationdate::text as "registrationDate", + a.compatibility as compatibility, + a.last_collection_date as "lastCollectionDate", + a.last_collection_total as "lastCollectionTotal" +from + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate is not null + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true +order by d.registrationdate desc +limit ?; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql new file mode 100644 index 00000000..e014461c --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql @@ -0,0 +1,13 @@ +select count(d.id) as count +from + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate >= cast(? as date) + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..1af6acd7 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql @@ -0,0 +1,14 @@ +select count(d.id) as count +from + dsm_services d + left outer join dsm_api a on (d.id = a.service) + left outer join dsm_service_organization dso on (d.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +where + d.registrationdate >= cast(? as date) + and d.eosc_datasource_type like ? + and d.registrationdate < a.last_collection_date + and d.registeredby is not null + and d.managed = true + and a.last_collection_total > 0 + and a.active = true; diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st new file mode 100644 index 00000000..3d4a236e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_header.st @@ -0,0 +1,4 @@ + + + + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st new file mode 100644 index 00000000..18f23df4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_project.st @@ -0,0 +1,4 @@ + + $p.code$ - $p.acronym$ - $p.title$ + $p.idnamespace$ + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st new file mode 100644 index 00000000..2a263139 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_dspace_tail.st @@ -0,0 +1,2 @@ + + diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st new file mode 100644 index 00000000..f0d21f1e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/st/projects_eprints.st @@ -0,0 +1 @@ +$p.acronym$ - $p.title$
  • $p.code$ - $p.acronym$ - $p.title$
    • $p.idnamespace$
    • diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery new file mode 100644 index 00000000..7f254339 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findCommunityContexts.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'community' or ./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'ri'] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery new file mode 100644 index 00000000..871c752a --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfiles.xquery @@ -0,0 +1,2 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery new file mode 100644 index 00000000..e84fafad --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[%s] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery new file mode 100644 index 00000000..d1bfe051 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findFunderContexts.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = 'funding'] +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery new file mode 100644 index 00000000..bba15af0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery @@ -0,0 +1,11 @@ +distinct-values( + let $format := collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public']]/PROPERTY[@key = "mdformat"]/@value/string() + + for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType') + where + $x//METADATA_FORMAT = $format and + $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and + $x//METADATA_FORMAT_LAYOUT = 'index' + return + concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format, ' @@@ ', $format, '-index-openaire') +) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery new file mode 100644 index 00000000..f5c10374 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findObjectStore.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') +where $x[./RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID = "%s"] +return $x/RESOURCE_PROFILE/BODY/CONFIGURATION/INTERFACES/INTERFACE[./@compliance="files"]/INTERFACE_EXTRA_FIELD[./@name="last_download_objId"]/text() diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery new file mode 100644 index 00000000..ba3ceab2 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery @@ -0,0 +1 @@ +distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string()) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery new file mode 100644 index 00000000..927632cf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/xquery/getRepoProfile.xquery @@ -0,0 +1,3 @@ +for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') +where $x/RESOURCE_PROFILE/BODY/CONFIGURATION/DATASOURCE_ORIGINAL_ID[@provenance="OPENAIRE"]/text() = "%s" +return $x diff --git a/apps/dnet-exporter-api/src/main/resources/global.properties b/apps/dnet-exporter-api/src/main/resources/global.properties new file mode 100644 index 00000000..bef55ff0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/global.properties @@ -0,0 +1,105 @@ +services.is.host = localhost +services.is.port = 8280 +services.is.protocol = http +services.is.context = app +services.is.baseurl = ${services.is.protocol}://${services.is.host}:${services.is.port}/${services.is.context}/services + +openaire.exporter.isLookupUrl = ${services.is.baseurl}/isLookUp +openaire.exporter.objectStoreServiceUrl = ${services.is.baseurl}/objectStore +openaire.exporter.isRegistryServiceUrl = ${services.is.baseurl}/isRegistry + +openaire.exporter.requestWorkers = 10 +openaire.exporter.requestTimeout = 10 + +openaire.exporter.cxfClientConnectTimeout = 60000 +openaire.exporter.cxfClientReceiveTimeout = 120000 + +# JDBC +openaire.exporter.jdbc.url = jdbc:postgresql://localhost:5432/dnet_openaireplus +openaire.exporter.jdbc.user = dnetapi +openaire.exporter.jdbc.pwd = dnetPwd +openaire.exporter.jdbc.minIdle = 1 +openaire.exporter.jdbc.maxIdle = 20 +openaire.exporter.jdbc.maxRows = 100 + +# PROJECTS +openaire.exporter.project.dspaceHeadTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_header.st +openaire.exporter.project.dspaceTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_project.st +openaire.exporter.project.dspaceTailTemplate = classpath:/eu/dnetlib/openaire/st/projects_dspace_tail.st +openaire.exporter.project.eprintsTemplate = classpath:/eu/dnetlib/openaire/st/projects_eprints.st +openaire.exporter.project.tsvFields = Grant Agreement Number, Project Acronym, Project Title, Call ID, Start Date, End Date, OA Mandate on Publications, OA Mandate on Datasets, Discipline, Organization, Country, Role, Person Name, Person Second Names, Person Email +openaire.exporter.project.projectsFundingQueryTemplate=classpath:/eu/dnetlib/openaire/sql/projects_fundings.sql.st +openaire.exporter.project.flushSize = 1000 + +# DATSOURCES +openaire.exporter.datasource.title = Data Sources +openaire.exporter.datasource.mongoHost = localhost +openaire.exporter.datasource.mongoPort = 27017 +openaire.exporter.datasource.mongoConnectionsPerHost = 10 +openaire.exporter.datasource.mongoCollectionName = wf_logs +openaire.exporter.datasource.mongoDbName = dnet_logs_prod +openaire.exporter.datasource.mongoQueryLimit= 100 +openaire.exporter.findSolrIndexUrl = /eu/dnetlib/openaire/xquery/findSolrIndexUrl.xquery +openaire.exporter.findIndexDsInfo = /eu/dnetlib/openaire/xquery/findIndexDsInfo.xquery +openaire.exporter.findObjectStore = /eu/dnetlib/openaire/xquery/findObjectStore.xquery +openaire.exporter.findFunderContexts = /eu/dnetlib/openaire/xquery/findFunderContexts.xquery +openaire.exporter.findCommunityContexts = /eu/dnetlib/openaire/xquery/findCommunityContexts.xquery +openaire.exporter.findContextProfiles = /eu/dnetlib/openaire/xquery/findContextProfiles.xquery +openaire.exporter.findContextProfilesByType = /eu/dnetlib/openaire/xquery/findContextProfilesByType.xquery +openaire.exporter.getRepoProfile = /eu/dnetlib/openaire/xquery/getRepoProfile.xquery + +openaire.exporter.contentLoadQuery = { "$and" : [ { "system:profileName" : "Graph construction [PROD]" }, { "system:isCompletedSuccessfully" : "true" }, { "reuseContent" : "false" } ] } + +# REST API CONFIGURATION +openaire.exporter.swaggerDsm.apiTitle = OpenAIRE aggregator REST API +openaire.exporter.swaggerDsm.apiDescription = The OpenAIRE data provision REST API allows developers to access the metadata information space of OpenAIRE programmatically. +openaire.exporter.swaggerDsm.apiLicense = LICENSED UNDER GNU AFFERO GENERAL PUBLIC LICENSE. +openaire.exporter.swaggerDsm.apiLicenseUrl = https://www.gnu.org/licenses/agpl-3.0.txt +openaire.exporter.swaggerDsm.apiContacName = D-Net team +openaire.exporter.swaggerDsm.apiContactUrl = http://www.openaire.eu +openaire.exporter.swaggerDsm.apiContactEmail = dnet-team@isti.cnr.it + +openaire.exporter.swaggerProjects.apiTitle = OpenAIRE projects REST API +openaire.exporter.swaggerProjects.apiDescription = The OpenAIRE projects REST API allows programmatic access to funded research projects metadata. +openaire.exporter.swaggerProjects.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerProjects.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerProjects.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerProjects.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerProjects.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerFunders.apiTitle = OpenAIRE funders REST API +openaire.exporter.swaggerFunders.apiDescription = The OpenAIRE funders REST API allows programmatic access to the funding agencies metadata in OpenAIRE. +openaire.exporter.swaggerFunders.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerFunders.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerFunders.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerFunders.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerFunders.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerCommunities.apiTitle = OpenAIRE Communities REST API +openaire.exporter.swaggerCommunities.apiDescription = The OpenAIRE communities REST API allows programmatic access to the communities configurations in OpenAIRE. +openaire.exporter.swaggerCommunities.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerCommunities.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerCommunities.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerCommunities.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerCommunities.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerContexts.apiTitle = OpenAIRE Contexts REST API +openaire.exporter.swaggerContexts.apiDescription = The OpenAIRE contexts REST API allows programmatic access to the context profiles in OpenAIRE. +openaire.exporter.swaggerContexts.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerContexts.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerContexts.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerContexts.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerContexts.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +openaire.exporter.swaggerInfo.apiTitle = OpenAIRE Info REST API +openaire.exporter.swaggerInfo.apiDescription = The OpenAIRE Info REST API allows programmatic access to some interesting dates related to the content indexed in OpenAIRE. +openaire.exporter.swaggerInfo.apiLicense = ${openaire.exporter.swaggerDsm.apiLicense} +openaire.exporter.swaggerInfo.apiLicenseUrl = ${openaire.exporter.swaggerDsm.apiLicenseUrl} +openaire.exporter.swaggerInfo.apiContacName = ${openaire.exporter.swaggerDsm.apiContacName} +openaire.exporter.swaggerInfo.apiContactUrl = ${openaire.exporter.swaggerDsm.apiContactUrl} +openaire.exporter.swaggerInfo.apiContactEmail = ${openaire.exporter.swaggerDsm.apiContactEmail} + +# VOCABULARIES +openaire.exporter.vocabularies.baseUrl = http://localhost:8980/provision/mvc/vocabularies +openaire.exporter.vocabularies.countriesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:countries.json +openaire.exporter.vocabularies.datasourceTypologiesEndpoint = ${openaire.exporter.vocabularies.baseUrl}/dnet:eosc_datasource_types.json diff --git a/apps/dnet-exporter-api/src/main/resources/logback-spring.xml b/apps/dnet-exporter-api/src/main/resources/logback-spring.xml new file mode 100644 index 00000000..756240fa --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java new file mode 100644 index 00000000..43974fd9 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/CommunityApiControllerTest.java @@ -0,0 +1,63 @@ +package eu.dnetlib.openaire.community; + +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.util.Date; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +/** + * Created by Alessia Bardi on 2019-04-04. + * + * @author Alessia Bardi + */ +@SpringBootTest +@WebMvcTest(CommunityApiController.class) +public class CommunityApiControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private CommunityApiController communityController; + + @Test + public void testListCommunities() throws Exception { + final CommunitySummary cs = new CommunitySummary(); + cs.setDescription("the description"); + cs.setId("id1"); + cs.setLastUpdateDate(new Date()); + cs.setName("X"); + cs.setShortName("x"); + final List csList = singletonList(cs); + given(communityController.listCommunities()).willReturn(csList); + + mvc.perform(get("/community/communities").contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$[0].description", is(cs.getDescription()))) + .andExpect(jsonPath("$[0].id", is(cs.getId()))) + // TODO verify the lastUpdateDate format + // .andExpect(jsonPath("$[0].lastUpdateDate", is(cs.getLastUpdateDate()))) + .andExpect(jsonPath("$[0].name", is(cs.getName()))) + .andExpect(jsonPath("$[0].shortName", is(cs.getShortName()))); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java new file mode 100644 index 00000000..0d9420ff --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/selectioncriteria/SelectionCriteriaTest.java @@ -0,0 +1,77 @@ +package eu.dnetlib.openaire.community.selectioncriteria; + +import com.google.gson.Gson; +import eu.dnetlib.data.bulktag.selectioncriteria.Selection; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.ArrayList; +import java.util.List; + + +public class SelectionCriteriaTest { + + private final String json = "{\"selectioncriteria\": " + + "{ \"criteria\": [ { \"constraint\": " + + "[ { \"field\": \"contributor\", \"value\": \"Neuroinformatics\", \"verb\": \"contains\" } ] } ] }"+ + "}"; + + + @Test + public void loadSelectionCriteria(){ + SelectionCriteria sc = new Gson().fromJson(json, SelectionCriteria.class); + + + + } + + @Test + public void printSelectionCriteria(){ + SelectionCriteria sc = new SelectionCriteria(); + + List list_constraints = new ArrayList<>(); + + Constraints constraints = new Constraints(); + + List list_constraint = new ArrayList<>(); + Constraint c = new Constraint(); + c.setField("fake"); + c.setValue("fake"); + c.setVerb("fake"); + list_constraint.add(c); + c = new Constraint(); + c.setField("fake1"); + c.setValue("fake1"); + c.setVerb("fake1"); + list_constraint.add(c); + constraints.setConstraint(list_constraint); + list_constraints.add(constraints); + + constraints = new Constraints(); + list_constraint = new ArrayList<>(); + c = new Constraint(); + c.setField("fake"); + c.setValue("fake"); + c.setVerb("fake"); + list_constraint.add(c); + c = new Constraint(); + c.setField("fake1"); + c.setValue("fake1"); + c.setVerb("fake1"); + list_constraint.add(c); + constraints.setConstraint(list_constraint); + + + list_constraints.add(constraints); + sc.setCriteria(list_constraints); + + System.out.println(new Gson().toJson(sc)); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java new file mode 100644 index 00000000..3a30f10f --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/context/ContextMappingUtilsTest.java @@ -0,0 +1,42 @@ +package eu.dnetlib.openaire.context; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Created by Alessia Bardi on 2019-04-04. + * + * + * @author Alessia Bardi + */ +public class ContextMappingUtilsTest { + + @Test + // @Disabled + public void testParseContextProfile() throws IOException { + final String communityProfile = IOUtils.toString(getClass().getResourceAsStream("community_test.xml"), Charset.defaultCharset()); + final Context context = ContextMappingUtils.parseContext(communityProfile, new LinkedBlockingQueue<>()); + + assertNotNull(context); + assertNotNull(context.getId()); + + final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + final String json = gson.toJson(context); + assertNotNull(json); + assertFalse(json.isEmpty()); + + System.out.println(gson.toJson(gson.fromJson(json, Context.class))); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java new file mode 100644 index 00000000..39a6875d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/dsm/DatasourceApiControllerTest.java @@ -0,0 +1,70 @@ +package eu.dnetlib.openaire.dsm; + +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.core.Is.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils; +import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; +import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; +import eu.dnetlib.openaire.vocabularies.Country; + +@SpringBootTest +@WebMvcTest(DsmApiController.class) +public class DatasourceApiControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private DsmApiController dsController; + + @Test + // @Disabled + public void listCountries() throws Exception { + final Country c = new Country("it", "Italy"); + final List countries = singletonList(c); + + given(dsController.listCountries()).willReturn(countries); + + mvc.perform(get("/ds/countries") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(1))) + .andExpect(jsonPath("$[0].code", is(c.getCode()))) + .andExpect(jsonPath("$[0].name", is(c.getName()))); + } + + @Test + public void testAddDs() throws DsmException { + final DatasourceDetails dd = new DatasourceDetails().setId("openaire____::issn19718357") + .setIssn("1971-8357") + .setNamespaceprefix("19718357____"); + + final DatasourceDbEntry datasourceDbEntry = DsmMappingUtils.asDbEntry(dd); + System.out.println(datasourceDbEntry.getIssn()); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java new file mode 100644 index 00000000..3e0ca5af --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java @@ -0,0 +1,50 @@ +package eu.dnetlib.openaire.funders; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import eu.dnetlib.openaire.context.Context; +import eu.dnetlib.openaire.context.ContextMappingUtils; +import eu.dnetlib.openaire.funders.domain.FunderDetails; + +public class FunderContextClientTest { + + private FunderDao fDao; + + @BeforeEach + public void setUp() { + fDao = new FunderDao(); + } + + @Test + public void testParseContextProfile() throws IOException { + final String contextProfile = IOUtils.toString(getClass().getResourceAsStream("ec-fp7.xml"), Charset.defaultCharset()); + final Context context = ContextMappingUtils.parseContext(contextProfile, new LinkedBlockingQueue<>()); + + assertNotNull(context); + assertNotNull(context.getId()); + + final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + final String json = gson.toJson(context); + assertNotNull(json); + assertFalse(json.isEmpty()); + + // System.out.println(gson.toJson(gson.fromJson(json, Context.class))); + + final FunderDetails funderDetails = ContextMappingUtils.asFunderDetails(context); + // System.out.println(gson.toJson(funderDetails)); + + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java new file mode 100644 index 00000000..746fa144 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/info/InfoControllerTest.java @@ -0,0 +1,119 @@ +package eu.dnetlib.openaire.info; + +import static org.hamcrest.core.Is.is; +import static org.mockito.BDDMockito.given; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.nio.charset.Charset; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.assertj.core.util.Maps; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +@SpringBootTest +@WebMvcTest(InfoController.class) +public class InfoControllerTest { + + public static final MediaType APPLICATION_JSON_UTF8 = new MediaType( + MediaType.APPLICATION_JSON.getType(), + MediaType.APPLICATION_JSON.getSubtype(), + Charset.forName("utf8")); + + @Autowired + private MockMvc mvc; + + @MockBean + private InfoController infoController; + + private LocalDate expectedDate; + private String formattedDate; + + @BeforeEach + public void setup() { + expectedDate = LocalDate.now(); + formattedDate = expectedDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.claim_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.oaf_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.odf_load_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.inference_date.name())).willReturn(expectedDate); + given(infoController.getDate(JdbcInfoDao.DATE_INFO.stats_update_date.name())).willReturn(expectedDate); + given(infoController.listInfo()).willReturn(Maps.newHashMap(JdbcInfoDao.DATE_INFO.inference_date.name(), LocalDate.now())); + given(infoController.listInfoKeys()) + .willReturn(Arrays.stream(JdbcInfoDao.DATE_INFO.values()).map(JdbcInfoDao.DATE_INFO::name).collect(Collectors.toList())); + + } + + @Test + public void testGetClaimDate() throws Exception { + mvc.perform(get("/info/claim_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetOafLoadDate() throws Exception { + mvc.perform(get("/info/oaf_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetOdfLoadDate() throws Exception { + mvc.perform(get("/info/odf_load_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetInferenceDate() throws Exception { + mvc.perform(get("/info/inference_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testGetStatsUpdateDateDate() throws Exception { + mvc.perform(get("/info/stats_update_date") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is(formattedDate))); + } + + @Test + public void testListAllDates() throws Exception { + System.out.println(mvc.perform(get("/info") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString()); + + } + + @Test + public void listKeys() throws Exception { + System.out.println(mvc.perform(get("/info/keys") + .contentType(APPLICATION_JSON_UTF8)) + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString()); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java new file mode 100644 index 00000000..5524289e --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectQueryParamsTest.java @@ -0,0 +1,41 @@ +package eu.dnetlib.openaire.project; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class ProjectQueryParamsTest { + + private ProjectQueryParams queryParams; + + @BeforeEach + public void setUp() throws Exception { + queryParams = new ProjectQueryParams(); + + } + + @Test + public void testVerifyParamWhiteSpace() { + queryParams.verifyParam("Discovery Projects"); + } + + @Test + public void testVerifyParamPercentage() { + queryParams.verifyParam("Discovery%20Projects"); + } + + @Test + public void testVerifyDateParam() { + final String correctDate = "2012-03-04"; + assertEquals(correctDate, queryParams.verifyDateParam(correctDate)); + + } + + @Test + public void testVerifyDateParamException() { + final String wrongDate = "12-12-12"; + assertThrows(IllegalArgumentException.class, () -> queryParams.verifyDateParam(wrongDate)); + } +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java new file mode 100644 index 00000000..efdcea8a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/ProjectsControllerTest.java @@ -0,0 +1,131 @@ +package eu.dnetlib.openaire.project; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; + +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +@Disabled +@SpringBootTest +public class ProjectsControllerTest { + + private static final Log log = LogFactory.getLog(ProjectsControllerTest.class); + private final String queryTemplate = "/eu/dnetlib/openaire/sql/projects_fundings.sql.st"; + + private final Resource expectedQueryTemplate = new ClassPathResource("/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st"); + + private ProjectsController controller; + private ProjectQueryParams params; + + @BeforeEach + public void setup() { + controller = new ProjectsController(); + final Resource template = new ClassPathResource(queryTemplate); + + // TODO reimplement bean injection for testing + // controller.setProjectsFundingQueryTemplate(template); + params = new ProjectQueryParams(); + } + + @Test + public void testObtainFP7Query() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7"); + log.debug(res); + log.debug(st); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QuerySP1() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("SP1"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QueryHealth() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("SP1::HEALTH"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::SP1::HEALTH"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFP7QueryHealth2() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FP7"); + params.setFundingPath("%::HEALTH"); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "ec__________::EC::FP7::%::HEALTH"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainWellcomeTrustQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("WT"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "wt__________::WT"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testObtainFCTQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("FCT"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "fct_________::FCT"); + log.debug(res); + assertEquals(st.toString(), res); + } + + @Test + public void testQueryWithDateParams() throws IllegalArgumentException, IOException { + params.setFundingProgramme("WT"); + params.setFundingPath(null); + params.setStartFrom("2015-05-04"); + final String res = controller.obtainQuery(params); + log.debug(res); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "wt__________::WT"); + final String q = st.toString() + " AND startdate >= '2015-05-04'"; + assertEquals(q, res); + } + + @Test + public void testObtainSNSFQuery() throws IllegalArgumentException, IOException { + params.setFundingProgramme("SNSF"); + params.setFundingPath(null); + final String res = controller.obtainQuery(params); + final StringTemplate st = new StringTemplate(IOUtils.toString(expectedQueryTemplate.getInputStream(), ProjectsController.UTF8)); + st.setAttribute("fundingprefix", "snsf________::SNSF"); + log.debug(res); + assertEquals(st.toString(), res); + } + +} diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java new file mode 100644 index 00000000..7d342de5 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/project/domain/ProjectDetailsTest.java @@ -0,0 +1,26 @@ +package eu.dnetlib.openaire.project.domain; + +import java.io.IOException; + +import eu.dnetlib.openaire.project.domain.db.ProjectDetails; +import org.junit.jupiter.api.Test; + +/** + * Created by claudio on 05/07/2017. + */ +public class ProjectDetailsTest { + + @Test + public void testCSV() throws IOException { + + final ProjectDetails p = ProjectDetails.fromCSV( + "arc_________::ANZCCART,,ANZCCART,{},\"[\"\"\\u003cfundingtree\\u003e\\n \\u003cfunder\\u003e\\n \\u003cid\\u003earc_________::ARC\\u003c/id\\u003e\\n \\u003cshortname\\u003eARC\\u003c/shortname\\u003e\\n \\u003cname\\u003eAustralian Research Council (ARC)\\u003c/name\\u003e\\n \\u003cjurisdiction\\u003eAU\\u003c/jurisdiction\\u003e\\n \\u003c/funder\\u003e\\n \\u003cfunding_level_0\\u003e\\n \\u003cid\\u003earc_________::ARC::Special Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/id\\u003e\\n \\u003cname\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/name\\u003e\\n \\u003cdescription\\u003eSpecial Research initiative (Australian and New Zealand Council for the Care of Animals in Research and Teaching)\\u003c/description\\u003e\\n \\u003cparent/\\u003e\\n \\u003cclass\\u003earc:fundingStream\\u003c/class\\u003e\\n \\u003c/funding_level_0\\u003e\\n \\u003c/fundingtree\\u003e\"\"]\""); + + System.out.println(p.asJson()); + + System.out.println(p.asCSV()); + + + } + +} diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml new file mode 100644 index 00000000..8508aea9 --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/context/community_test.xml @@ -0,0 +1,838 @@ + +
    + + + + + +
    + + + + manager + The scope of this community is to provide access to publications, research data, projects and software that are related to agricultural and food sciences + https://pbs.twimg.com/profile_images/837614594881384448/-Jr9Fpst_400x400.jpg + Agricultural and Food Sciences + pzervas@agroknow.com + animal production and health,fisheries and aquaculture,food safety and human nutrition,information management,food technology,agri-food education and extension,natural resources and environment,food system,engineering technology and Research,agriculture,food safety risk assessment,food security,farming practices and systems,plant production and protection,agri-food economics and policy,food distribution,forestry + AGINFRA+ - Accelerating user-driven e-infrastructure innovation in Food & Agriculture has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 731001. + oac_aginfra + 2018-03-01T12:00:00 + + + Strengthening European Food Chain Sustainability by Quality and Procurement Policy + 678024 + https://www.strength2food.eu/ + H2020 + EC + Strength2Food + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 678024. + https://www.strength2food.eu/publications/ + + + Future Internet Business Collaboration Networks in Agri-Food, Transport and Logistics + 604123 + https://www.fispace.eu/ + FP7 + EC + FIspace + FIspace has received funding from the European Commission under the FP7 programme. + https://www.fispace.eu/publications.html + + + Personalised public services in support of the implementation of the Common Agricultural Policy + 693171 + https://www.recap-h2020.eu/ + H2020 + EC + RECAP + This project has received funding from the European Union's Horizon 2020 Research and Innovation Programme under Grant Agreement No. 693171. + + + Metrics, Models and Foresight for European SUStainable Food And Nutrition Security + 633692 + http://www.susfans.org/ + H2020 + EC + SUSFANS + + http://www.susfans.org/portfolio + + + Integrated and innovative key actions for mycotoxin management in the food and feed chain + 678781 + http://www.mycokey.eu/ + H2020 + EC + MycoKey + + + Smart Food and Agribusiness: Future Internet for Safe and Healthy Food from Farm to Fork + 285326 + http://smartagrifood.eu/ + FP7 + EC + SmartAgriFood + SmartAgriFood is funded by the EU Seventh Framework Programme under the FI.ICT-2011.1.8 Work Programme + + + Tools for Assessment and Planning of Aquaculture Sustainability + 678396 + http://tapas-h2020.eu/ + H2020 + EC + TAPAS + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 678396. + http://tapas-h2020.eu/home-publications/ + + + Next generation, Cost-effective, Compact, Multifunctional Web Enabled Ocean Sensor Systems Empowering Marine, Maritime and Fisheries Management + 614102 + http://www.nexosproject.eu/ + FP7 + EC + NeXOS + NeXOS has received funding from the European Union’s Seventh Programme for research, technological development and demonstration under grant agreement No 614102 + http://www.nexosproject.eu/dissemination/publications + + + Robot Fleets for Highly Effective Agriculture and Forestry Management + 245986 + http://www.rhea-project.eu/ + FP7 + EC + RHEA + The research leading to these results has received funding from the European Union’s Seventh Framework Programme [FP7/2007-2013] under Grant Agreement nº 245986 + http://www.rhea-project.eu/Workshops/Workshop1/RHEA-2011-_Proceedings_of_the_1st_RHEA-WorkShop.pdf + + + Optimizing the management and sustainable use of forest genetic resources in Europe + 676876 + http://www.gentree-h2020.eu/ + H2020 + EC + GENTREE + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 676876. + http://www.gentree-h2020.eu/resources/publications/ + + + Distributed, Integrated and Harmonised Forest Information for Bioeconomy Outlooks + 633464 + http://diabolo-project.eu/ + H2020 + EC + DIABOLO + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 633464. + http://diabolo-project.eu/category/research-papers/ + + + PROVIding smart DElivery of public goods by EU agriculture and forestry + 633838 + http://www.provide-project.eu/ + H2020 + EC + PROVIDE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 633838. + + + Agricultural Low Cost Integral System of nodes with communication networks for remote water management with sensors and monitoring of the vegetative state of crops + 618123 + http://www.ict-agri.eu/node/38662 + FP7 + EC + ICT-AGRI + This project has received funding from the European Union’s Seventh Framework Programme for research, technological development and demonstration under grant agreement no 618123 [ICT-AGRI 2]. + + + LAND Management: Assessment, Research, Knowledge base + 635201 + http://landmark2020.eu/ + H2020 + EC + LANDMARK + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 635201. + http://landmark2020.eu/publication-trees/ + + + Soil Care for profitable and sustainable crop production in Europe + 677407 + https://www.soilcare-project.eu/ + H2020 + EC + SOILCARE + The SOILCARE project is co-funded by European Commission, Directorate General for Research under Framework Programme HORIZON 2020, [H2020-SFS-2015-2] (Contract Number [677407]) + + + Innovative tools enabling drinking WATER PROTECTion in rural and urban environments + 727450 + https://water-protect.eu/ + H2020 + EC + WATERPROTECT + + + + 654182 + http://www.envriplus.eu/ + H2020 + EC + ENVRIplus + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 654182. + http://www.envriplus.eu/publications/ + + + Internet of Food and Farm 2020 + 731884 + https://www.iof2020.eu/ + H2020 + EC + IOF2020 + IoF2020 has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement no. 731884 + + + Sweet Pepper Harvesting Robot + 644313 + http://www.sweeper-robot.eu/ + H2020 + EC + SWEEPER + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 644313. + + + Future Internet Enabled Agricultural Applications + 632874 + http://fractals-fp7.com/ + FP7 + EC + FRACTALS + + + + Low-cost, hand-held, and non-invasive optical sensor for multiparametric field analysis of grapes and leaves in vineyards + 262011 + FP7 + EC + PREMIVM + The research leading to these results has received funding from the European Community’s Seventh Framework Programme FP7/2007-2013 managed by REA-Research Executive Agency under grant agreement n° 262011 + + + Space for Agricultural Innovation + 652642 + http://agrispin.eu/ + H2020 + EC + AGRISPIN + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 652642 + http://agrispin.eu/publicaons/ + + + Authentication and Authorisation for Research and Collaboration + 730941 + https://aarc-project.eu/ + H2020 + EC + AARC2 + AARC2 is funded by the European Union’s Horizon 2020 research and innovation programme under Grant Agreement 730941. + + + European Open Science Cloud - pilot + 739563 + https://eoscpilot.eu/ + H2020 + EC + EOSC-pilot + EOSCPilot.eu has received funding from the European Commission’s Horizon 2020 research and innovation programme under the Grant Agreement no 739563. + + + BUSINESS INTELLIGENCE SERVICE FOR THE MANAGEMENT OF CROPS BASED ON CLOUD AND BIG DATA + 672453 + H2020 + EC + BYNSE + + + + Demonstration of a cloud-based precision farming management system for a sustainable and intensive agriculture to secure long-term food supply in Europe + 672655 + H2020 + EC + AgriCloud + + + + Demonstration of a cloud-based precision farming management system for a sustainable and intensive agriculture to secure long-term food supply in Europe - Phase II + 720176 + H2020 + EC + AgriCloud P2 + + + + Data-Driven Bioeconomy + 732064 + https://www.databio.eu/en/ + H2020 + EC + DataBio + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 732064 + https://www.databio.eu/en/publications/ + + + Safe Food and Feed through an Integrated ToolBox for Mycotoxin Management + 678012 + https://www.mytoolbox.eu/ + H2020 + EC + MyToolBox + This project has received funding from the European Union's Horizon 2020 research and innovation programme (link is external) under grant agreement No 678012. + https://www.mytoolbox.eu/publications/papers + + + Drone-based integrated monitoring system for early detection of crop pathology and pest control in high tech greenhouse agriculture + 697900 + H2020 + EC + GIDROM + + + + Embedding crop diversity and networking for local high quality food systems + 633571 + http://www.diversifood.eu/ + H2020 + EC + DIVERSIFOOD + This project received funding from the European Union's Horizon 2020 Research and Innovation program under Grant Agreement n° 633571 + http://www.diversifood.eu/publications/ + + + Traditional tomato varieties and cultural practices: a case for agricultural diversification with impact on food security and health of European population + 634561 + http://traditom.eu/ + H2020 + EC + TRADITOM + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 634561. + http://traditom.eu/publications/ + + + Linking genetic resources, genomes and phenotypes of Solanaceous crops + 677379 + http://www.g2p-sol.eu/ + H2020 + EC + G2P-SOL + The G2P-SOL project (Title: Linking genetic resources, genomes and phenotypes of Solanaceous crops) has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 677379. + http://www.g2p-sol.eu/Publications.html + + + Improve performance of organic agriculture by boosting organic seed and plant breeding efforts across Europe + 727230 + https://www.liveseed.eu/ + H2020 + EC + LIVESEED + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727230 and by the Swiss State Secretariat for Education, Research and Innovation under contract number 17.00090. + https://www.liveseed.eu/resources/scientific-articles/ + + + Transition paths to sustainable legume based systems in Europe + 727973 + https://www.true-project.eu/ + H2020 + EC + TRUE + TRansition paths to sUstainable legume-based systems in Europe (TRUE) has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No. 727973 + https://www.true-project.eu/resources-links/linked-resources/ + + + Innovine Project + 311775 + http://www.innovine.eu/home.html + FP7 + EC + INNOVINE + Innovine is a European collaborative project that has received funding from the European Union’s Seventh Framework Programme for research, technological development and demonstration under grant agreement n° 311775. + http://www.innovine.eu/publications-ressources.html + + + 633531 + https://www.feed-a-gene.eu/ + H2020 + EC + FEED-A-GENE + The Feed-a-Gene Project has received funding from the European Union’s H2020 Programme under grant agreement no 633531. + https://www.feed-a-gene.eu/media/scientific-papers + + + Multidisciplinary Approach to Practical and Acceptable Precision Livestock Farming for SMEs in Europe and world-wide + 227138 + FP7 + EC + BrightAnimal + + + + Practical implementation of precision livestock technologies and services at European pig farms using the living lab methodology + 311989 + FP7 + EC + ALL-SMART-PIGS + + + + Innovative and sustainable systems combining automatic milking and precision grazing + 314879 + https://autograssmilk.dk/ + FP7 + EC + AUTOGRASSMILK + This project has received funding from the European Union's Seventh Framework Programme managed by REA-Research Executive Agency [FP7/2007-2013] under grant agreement no. SME-2012-2-314879. + https://autograssmilk.dk/litterature/ + + + DIVERSITY OF LOCAL PIG BREEDS AND PRODUCTION SYSTEMS FOR HIGH QUALITY TRADITIONAL PRODUCTS AND SUSTAINABLE PORK CHAINS + 634476 + https://treasure.kis.si/ + H2020 + EC + TREASURE + + https://treasure.kis.si/reviews-and-publications/ + + + Innovative Management of Animal Genetic Resources + 677353 + http://www.imageh2020.eu/ + H2020 + EC + IMAGE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 677353 + http://www.imageh2020.eu/conteudo.php?idm=22&lang=en + + + Genomic management Tools to Optimise Resilience and Efficiency + 727213 + https://www.gentore.eu/ + H2020 + EC + GenTORE + GenTORE is a Horizon 2020 project running from 1 June 2017 to 31 May 2022. This research received funding from the European Union's H2020 Research and Innovation Program under agreement No. 727213. + + + Towards an e-infrastructure Roadmap for Open Science in Agriculture + 730988 + http://www.erosa.aginfra.eu/ + H2020 + EC + e-ROSA + e-ROSA - e-infrastructure Roadmap for Open Science in Agriculture has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 730988. + http://www.erosa.aginfra.eu/publications + + + An Earth obseRvation Model based RicE information Service + 606983 + FP7 + EC + ERMES + + + + Sustainable techno-economic solutions for the agricultural value chain + 690142 + http://www.agrocycle.eu/ + H2020 + EC + AgroCycle + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 690142 + + + Innovative approaches to turn agricultural waste into ecological and economic assets + 688338 + http://noaw2020.eu/ + H2020 + EC + NoAW + The project leading to this application has funding from European Union’s Horizon 2020 research and innovation programme under grant agreement No 688338. + + + Bringing added value to agriculture and forest sectors by closing the research and innovation divide + 696394 + http://www.agriforvalor.eu/ + H2020 + EC + AGRIFORVALOR + This project has received funding from the European Union´s Horizon 2020 research and innovation programme under grant agreement No 696394. + http://www.agriforvalor.eu/downloads/ + + + NEFERTITI: Innovation in Demo Farms + 772705 + H2020 + EC + NEFERTITI + NEFERTITI has received funding from the European Union’s Horizon 2020 Programme for Research & Innovation under grant agreement n°772705. + + + Collective Awareness PlatformS for Environmentally-sound Land management based on data technoLogies and Agrobiodiversity + 688813 + http://www.capsella.eu/ + H2020 + EC + CAPSELLA + Capsella has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 688813 + http://www.capsella.eu/conference-papers/ + + + FArming Tools for external nutrient Inputs and water Management + 633945 + http://fatima-h2020.eu/ + H2020 + EC + FATIMA + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 633945. + + + Advisory platform for small farms based on earth observation + 687412 + http://apollo-h2020.eu/ + H2020 + EC + APOLLO + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 687412. + + + Data Driven Dairy Decisions 4 Farmers + 696367 + http://www.4d4f.eu/ + H2020 + EC + 4D4F + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 696367. + + + Aerial Data Collection and Analysis, and Automated Ground Intervention for Precision Farming + 644227 + http://flourish-project.eu/ + H2020 + EC + Flourish + The Flourish project is funded by the European Community's Horizon 2020 programme under grant agreement no 644227-Flourish and from the Swiss State Secretariat for Education, Research and Innovation (SERI) under contract number 15.0029. + http://flourish-project.eu/documents/ + + + Integration of Farm Management Information Systems to support real-time management decisions and compliance of management standards + 212117 + http://www.futurefarm.eu/ + FP7 + EC + FutureFarm + Funded by the Seventh Research Framework Programme (FP7) of the European Union under the Cooperation programme in the Food, Agriculture, Fisheries and Biotechnology theme. Grant Agreement No 212117 (Work Programme KBBE-2007-1-4-05 "The farm of tomorrow"). + http://www.futurefarm.eu/publications + + + WATER AND ENERGY ADVANCED MANAGEMENT FOR IRRIGATION + 619061 + FP7 + EC + WEAM4i + + + + A web-based system for real-time Monitoring and Decision Making for Integrated Vineyard Management + 262059 + http://www.modem-ivm.eu + FP7 + EC + MoDeM_IVM + MoDeM_IVM is funded by the European Union's Seventh Framework Programme managed by REA-Research Executive Agency ([FP7/2007-2013] [ FP7/2007-2011]) under grant agreement n° [262059] + + + Crop, Livestock and Forests Integrated System for Intelligent Automation, Processing and Control + 604659 + http://www.clafis-project.eu/ + FP7 + EC + CLAFIS + + http://www.clafis-project.eu/index.php/dissemination + + + Linked Open Earth Observation Data for Precision Farming + 611141 + http://www.linkedeodata.eu/ + FP7 + EC + LEO + + http://www.linkedeodata.eu/Publications + + + AUTONOMOUS CLOUD-COMPUTING VINEYARD ROBOT TO OPTIMISE YIELD MANAGEMENT AND WINE QUALITY + 605630 + http://vinbot.eu/ + FP7 + EC + VINBOT + + + + Online Professional Irrigation Scheduling Expert System + 613717 + http://www.opiris.eu/ + FP7 + EC + OpIRIS + + http://www.opiris.eu/index.php/publications/ + + + Development of an automatic irrigation and fertilization system + 286772 + FP7 + EC + OPTIFERT + + + + A precise irrigation sensor system to provide an accurate indication of water status in crops and deliver increased yields to farmers + 720032 + http://saturas-ag.com/ + H2020 + EC + StemSense + + + + 674786 + H2020 + EC + VitiPrecision 2020 + + + + Robot shoots herbicide only on weeds, reducing usage by more than 90% + 736354 + H2020 + EC + ASTERIX + + + + Subarea specific irrigation system for pivot- and linear fertigation techniques + 720184 + H2020 + EC + SMART Fertigation + + + + Novel sensor based soil-plant-climate control system for European smart farming + 717797 + H2020 + EC + SenSOP-II + + + + Smart Irrigation Control System with 40% Savings in Water for Universal Use + 720235 + http://yodfatengineers.com/ + H2020 + EC + IRRISAVE + + + + Real time and online monitoring of the debittering stage in the table olive processing + 697335 + http://www.global-olive.es/en/ + H2020 + EC + TELEOLIVA + + + + EARTH OBSERVATION FARMING + 650082 + H2020 + EC + EO-FARM + + + + Interactive Soil Quality Assessment in Europe and China for Agricultural Productivity and Environmental Resilience + 635750 + http://www.isqaper-project.eu/ + H2020 + EC + iSQAPER + iSQAPER is funded by: - The European Union’s Horizon 2020 Programme for research & innovation under grant agreement no 635750 - Ministry of Science and Technology, China (grant nr:2016YFE011270) - Chinese Academy of Sciences (grant nr:16146KYSB20150001) - Swiss State Secretariat for Education, Research and Innovation. Contract: 15.0170-1 + http://www.isqaper-project.eu/downloads/publications + + + Transfer of INNOvative techniques for sustainable WAter use in FERtigated crops + 689687 + http://www.fertinnowa.com/ + H2020 + EC + FERTINNOWA + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 689687 + + + Solutions for improving Agroecosystem and Crop Efficiency for water and nutrient use + 727247 + http://www.solace-eu.net/ + H2020 + EC + SolACE + This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 727247 (SolACE) + http://www.solace-eu.net/publications.html + + + Shared Innovation Space for Sustainable Productivity of Grasslands in Europe + 727368 + http://www.inno4grass.eu/en/ + H2020 + EC + Inno4Grass + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727368 + + + Peer-to-Peer Learning: Accessing Innovation through Demonstration + 727388 + https://www.plaid-h2020.eu/ + H2020 + EC + PLAID + This project has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 727388 (PLAID). + + + A data infrastructure to support agricultural scientific communities + AGINFRA + 283770 + EC + corda_______::be2f6f94e5c78b3fba34cbe8454a750f + + + Accelerating user-driven e-infrastructure innovation in Food Agriculture + AGINFRA PLUS + 731001 + EC + corda__h2020::558cf59efd8258c3e65083d750182257 + + + Designing InnoVative plant teams for Ecosystem Resilience and agricultural Sustainability + DIVERSify + 727284 + EC + corda__h2020::c1c2aec17185db77fcf6c8ecb1adc68e + + + + + + opendoar____::1a551829d50f1400b0dab21fdd969c04 + + Repository of the Faculty of Food Technology and Biotechnology + true + + + opendoar____::49af6c4e558a7569d80eee2e035e2bd7 + + CemOA + true + + + opendoar____::0266e33d3f546cb5436a10798e657d97 + + Organic Eprints + true + + + opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06 + + Agritrop + true + + + opendoar____::41bfd20a38bb1b0bec75acf0845530a7 + + Epsilon Open Archive + true + + + opendoar____::87ae6fb631f7c8a627e8e28785d9992d + + Opin Visindi + true + + + + + edenis + + + + efsa-pilot + + + + egene3 + + + + efsa-kj + + + + euromixproject + + + + discardless + + + + sedinstcjfst + + + + afinet-kc + + + + 2231-4784 + + + + 2231-0606 + + + + solace + + + + pa17 + + + + smartakis + + + + sedinstcjae + + + + phenology_camera + + + + + + + + +     \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml new file mode 100644 index 00000000..02c03c2d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FCT-project.xml @@ -0,0 +1,12 @@ + + + FCT + PT + 2003-10-31 + 32639 + Social Classes and «lifestyles» in the city of Oporto + fct_________::FCT::Orçamento de Funcionamento/POSC + PTDC/FIL-FIL/109889/2009 + 2000-11-01 + description of the funding path + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml new file mode 100644 index 00000000..33222e8d --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/FP7-project.xml @@ -0,0 +1,14 @@ + + + EC + EU + 2012-07-31 + 255646 + Semiconductor lasers for generation of non-diffracting (Bessel) beams. + ec__________::EC::FP7::SP3::PEOPLE + SENDBEAMS + 2010-08-01 + description of the funding path + + + \ No newline at end of file diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml new file mode 100644 index 00000000..695a154b --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/MSES-project.xml @@ -0,0 +1,11 @@ + + + MSES + 2009-12-31 + 001-0000000-3177 + Residence time of bacteria Escherichia coli in seawater and marine organisms + + irb_hr______::MSES::fundingStream + HR + 2007-01-01 + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml new file mode 100644 index 00000000..8081b73a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/SFI-project.xml @@ -0,0 +1,11 @@ + + + SFI + 2016-08-31 + 14/ADV/RC3022 + Dr. Vesna Jaksic + + sfi_________::SFI::Advance Award Programme + + 2014-09-01 + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml new file mode 100644 index 00000000..a5fd058c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/exporter/WT-project.xml @@ -0,0 +1,11 @@ + + WT + 2013-07-29 + 098241 + EYEDIO DIGI - DEVELOPMENT OF AN INNOVATIVE, AFFORDABLE, EASY TO USE, HANDHELD RETINAL IMAGING PRODUCT FOR THE DIAGNOSIS OF DIABETIC RETINOPATHY. + + wt__________::WT::Immunology and Infectious Disease + 2012-01-30 + + description of the funding path + diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml new file mode 100644 index 00000000..fa39af4a --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/funders/ec-fp7.xml @@ -0,0 +1,438 @@ + +
    + + + + + +
    + + + + PROVA + + FP7 + ec__________::EC::FP7 + ec:frameworkprogram + + SP3 + ec__________::EC::FP7::SP3 + ec:specificprogram + + PEOPLE + ec__________::EC::FP7::SP3::PEOPLE + ec:program + + + + SP1 + ec__________::EC::FP7::SP1 + ec:specificprogram + + ENERGY + ec__________::EC::FP7::SP1::ENERGY + ec:program + + + ENV + ec__________::EC::FP7::SP1::ENV + ec:program + + + TPT + ec__________::EC::FP7::SP1::TPT + ec:program + + + NMP + ec__________::EC::FP7::SP1::NMP + ec:program + + + SP1-JTI + ec__________::EC::FP7::SP1::SP1-JTI + ec:program + + + HEALTH + ec__________::EC::FP7::SP1::HEALTH + ec:program + + + ICT + ec__________::EC::FP7::SP1::ICT + ec:program + + + KBBE + ec__________::EC::FP7::SP1::KBBE + ec:program + + + SEC + ec__________::EC::FP7::SP1::SEC + ec:program + + + SSH + ec__________::EC::FP7::SP1::SSH + ec:program + + + SPA + ec__________::EC::FP7::SP1::SPA + ec:program + + + GA + ec__________::EC::FP7::SP1::GA + ec:program + + + + SP2 + ec__________::EC::FP7::SP2 + ec:specificprogram + + ERC + ec__________::EC::FP7::SP2::ERC + ec:program + + + + SP4 + ec__________::EC::FP7::SP4 + ec:specificprogram + + SME + ec__________::EC::FP7::SP4::SME + ec:program + + + INFRA + ec__________::EC::FP7::SP4::INFRA + ec:program + + + SiS + ec__________::EC::FP7::SP4::SiS + ec:program + + + REGIONS + ec__________::EC::FP7::SP4::REGIONS + ec:program + + + REGPOT + ec__________::EC::FP7::SP4::REGPOT + ec:program + + + INCO + ec__________::EC::FP7::SP4::INCO + ec:program + + + COH + ec__________::EC::FP7::SP4::COH + ec:program + + + + SP5 + ec__________::EC::FP7::SP5 + ec:specificprogram + + Fission + ec__________::EC::FP7::SP5::Fission + ec:program + + + Fusion + ec__________::EC::FP7::SP5::Fusion + ec:program + + + + UNKNOWN + ec__________::EC::FP7::UNKNOWN + ec:specificprogram + + UNKNOWN + ec__________::EC::FP7::UNKNOWN::UNKNOWN + ec:program + + + + + H2020 + ec__________::EC::H2020 + ec:h2020fundings + + MSCA-IF-EF-ST + ec__________::EC::H2020::MSCA-IF-EF-ST + ec:h2020toas + + + H2020-EEN-SGA + ec__________::EC::H2020::H2020-EEN-SGA + ec:h2020toas + + + RIA + ec__________::EC::H2020::RIA + ec:h2020toas + + + ERC + ec__________::EC::H2020::ERC + ec:h2020fundings + + ERC-POC + ec__________::EC::H2020::ERC::ERC-POC + ec:h2020toas + + + ERC-STG + ec__________::EC::H2020::ERC::ERC-STG + ec:h2020toas + + + ERC-ADG + ec__________::EC::H2020::ERC::ERC-ADG + ec:h2020toas + + + ERC-COG + ec__________::EC::H2020::ERC::ERC-COG + ec:h2020toas + + + ERC-LVG + ec__________::EC::H2020::ERC::ERC-LVG + ec:h2020toas + + + + MSCA-ITN-ETN + ec__________::EC::H2020::MSCA-ITN-ETN + ec:h2020toas + + + MSCA-IF-GF + ec__________::EC::H2020::MSCA-IF-GF + ec:h2020toas + + + SME-2 + ec__________::EC::H2020::SME-2 + ec:h2020toas + + + SESAR-IA + ec__________::EC::H2020::SESAR-IA + ec:h2020toas + + + MSCA-RISE + ec__________::EC::H2020::MSCA-RISE + ec:h2020toas + + + IA + ec__________::EC::H2020::IA + ec:h2020toas + + + CSA + ec__________::EC::H2020::CSA + ec:h2020toas + + + MSCA-COFUND-DP + ec__________::EC::H2020::MSCA-COFUND-DP + ec:h2020toas + + + SME-1 + ec__________::EC::H2020::SME-1 + ec:h2020toas + + + Shift2Rail-RIA + ec__________::EC::H2020::Shift2Rail-RIA + ec:h2020toas + + + CS2-IA + ec__________::EC::H2020::CS2-IA + ec:h2020toas + + + CS2-RIA + ec__________::EC::H2020::CS2-RIA + ec:h2020toas + + + MSCA-ITN-EID + ec__________::EC::H2020::MSCA-ITN-EID + ec:h2020toas + + + MSCA-IF-EF-RI + ec__________::EC::H2020::MSCA-IF-EF-RI + ec:h2020toas + + + FCH2-RIA + ec__________::EC::H2020::FCH2-RIA + ec:h2020toas + + + ERA-NET-Cofund + ec__________::EC::H2020::ERA-NET-Cofund + ec:h2020toas + + + SESAR-CSA + ec__________::EC::H2020::SESAR-CSA + ec:h2020toas + + + SESAR-RIA + ec__________::EC::H2020::SESAR-RIA + ec:h2020toas + + + BBI-RIA + ec__________::EC::H2020::BBI-RIA + ec:h2020toas + + + MSCA-COFUND-FP + ec__________::EC::H2020::MSCA-COFUND-FP + ec:h2020toas + + + CSA-LS + ec__________::EC::H2020::CSA-LS + ec:h2020toas + + + COFUND-PCP + ec__________::EC::H2020::COFUND-PCP + ec:h2020toas + + + ECSEL-RIA + ec__________::EC::H2020::ECSEL-RIA + ec:h2020toas + + + SGA-CSA + ec__________::EC::H2020::SGA-CSA + ec:h2020toas + + + MSCA-ITN-EJD + ec__________::EC::H2020::MSCA-ITN-EJD + ec:h2020toas + + + MSCA-IF-EF-CAR + ec__________::EC::H2020::MSCA-IF-EF-CAR + ec:h2020toas + + + BBI-IA-DEMO + ec__________::EC::H2020::BBI-IA-DEMO + ec:h2020toas + + + IMI2-RIA + ec__________::EC::H2020::IMI2-RIA + ec:h2020toas + + + MSCA-IF-EF-SE + ec__________::EC::H2020::MSCA-IF-EF-SE + ec:h2020toas + + + FCH2-IA + ec__________::EC::H2020::FCH2-IA + ec:h2020toas + + + BBI-IA-FLAG + ec__________::EC::H2020::BBI-IA-FLAG + ec:h2020toas + + + ECSEL-IA + ec__________::EC::H2020::ECSEL-IA + ec:h2020toas + + + BBI-CSA + ec__________::EC::H2020::BBI-CSA + ec:h2020toas + + + IMI2-CSA + ec__________::EC::H2020::IMI2-CSA + ec:h2020toas + + + PCP + ec__________::EC::H2020::PCP + ec:h2020toas + + + CS2-CSA + ec__________::EC::H2020::CS2-CSA + ec:h2020toas + + + FCH2-CSA + ec__________::EC::H2020::FCH2-CSA + ec:h2020toas + + + COFUND-EJP + ec__________::EC::H2020::COFUND-EJP + ec:h2020toas + + + SGA-RIA + ec__________::EC::H2020::SGA-RIA + ec:h2020toas + + + Shift2Rail-IA + ec__________::EC::H2020::Shift2Rail-IA + ec:h2020toas + + + Shift2Rail-CSA + ec__________::EC::H2020::Shift2Rail-CSA + ec:h2020toas + + + PPI + ec__________::EC::H2020::PPI + ec:h2020toas + + + COFUND-PPI + ec__________::EC::H2020::COFUND-PPI + ec:h2020toas + + + + + + + +     diff --git a/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st new file mode 100644 index 00000000..b75fcc9c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/sql/expected_projects_fundings.sql.st @@ -0,0 +1,11 @@ +SELECT + funder, + jurisdiction, + fundingpathid, + acronym, + title, + code, + startdate, + enddate +FROM projects_api +WHERE fundingpathid like '$fundingprefix$%' diff --git a/apps/dnet-exporter-api/src/test/resources/logback-test.xml b/apps/dnet-exporter-api/src/test/resources/logback-test.xml new file mode 100644 index 00000000..e369115c --- /dev/null +++ b/apps/dnet-exporter-api/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/apps/pom.xml b/apps/pom.xml index 382fea18..7abf14dc 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -16,6 +16,7 @@ dhp-broker-public-application dhp-mdstore-manager dnet-orgs-database-application + dnet-exporter-api scholexplorer-api