This commit is contained in:
Michele Artini 2022-12-12 15:55:35 +01:00
parent b37eb84e91
commit ee323bca6d
121 changed files with 5982 additions and 116 deletions

View File

@ -11,12 +11,12 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestParam;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.is.context.model.repository.ContextRepository;
import eu.dnetlib.is.resource.model.ResourceType;
import eu.dnetlib.is.resource.repository.ResourceTypeRepository;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.data.is.context.model.Context;
import eu.dnetlib.data.is.context.repository.ContextRepository;
import eu.dnetlib.data.is.resource.model.ResourceType;
import eu.dnetlib.data.is.resource.repository.ResourceTypeRepository;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.is.wfs.WfHistoryAjaxController;
@Controller

View File

@ -0,0 +1,54 @@
package eu.dnetlib.is;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(entityManagerFactoryRef = "entityManagerFactory", basePackages = {
"eu.dnetlib.data.is.context.repository",
"eu.dnetlib.data.is.resource.repository",
"eu.dnetlib.data.is.vocabulary.repository",
"eu.dnetlib.data.is.wf.repository"
})
public class MainDBConfig {
@Primary
@Bean(name = "dataSource")
@ConfigurationProperties(prefix = "spring.datasource")
public DataSource dataSource() {
return DataSourceBuilder.create().build();
}
@Primary
@Bean(name = "entityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
final EntityManagerFactoryBuilder builder,
@Qualifier("dataSource") final DataSource dataSource) {
return builder
.dataSource(dataSource)
.packages("eu.dnetlib.data.is.context.model", "eu.dnetlib.data.is.resource.model", "eu.dnetlib.data.is.vocabulary.model", "eu.dnetlib.data.is.wf.model")
.persistenceUnit("is")
.build();
}
@Primary
@Bean(name = "transactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("entityManagerFactory") final EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
}

View File

@ -0,0 +1,47 @@
package eu.dnetlib.is;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(entityManagerFactoryRef = "openaireEntityManagerFactory", transactionManagerRef = "openaireTransactionManager", basePackages = {
"eu.dnetlib.data.openaire.dsm.repository"
})
public class OpenaireDBConfig {
@Bean(name = "openaireDataSource")
@ConfigurationProperties(prefix = "openaire.datasource")
public DataSource dataSource() {
return DataSourceBuilder.create().build();
}
@Bean(name = "openaireEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean openaireEntityManagerFactory(
final EntityManagerFactoryBuilder builder,
@Qualifier("openaireDataSource") final DataSource dataSource) {
return builder
.dataSource(dataSource)
.packages("eu.dnetlib.data.openaire.dsm.model")
.persistenceUnit("openaire")
.build();
}
@Bean(name = "openaireTransactionManager")
public PlatformTransactionManager opeanaireTransactionManager(
@Qualifier("openaireEntityManagerFactory") final EntityManagerFactory openaireEntityManagerFactory) {
return new JpaTransactionManager(openaireEntityManagerFactory);
}
}

View File

@ -6,7 +6,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.data.is.context.model.Context;
public class AbstractContextController extends AbstractDnetController {

View File

@ -5,10 +5,10 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.is.context.model.Category;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.is.context.model.CtxChildInfo;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.context.model.Category;
import eu.dnetlib.data.is.context.model.Context;
import eu.dnetlib.data.is.context.model.CtxChildInfo;
import eu.dnetlib.is.errors.InformationServiceException;
@RestController
@RequestMapping("/ajax/contexts")

View File

@ -7,7 +7,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.node.ObjectNode;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.is.errors.InformationServiceException;
@RestController
@RequestMapping("/api/contexts")

View File

@ -12,15 +12,15 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import eu.dnetlib.is.context.model.Category;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.is.context.model.CtxChildInfo;
import eu.dnetlib.is.context.model.repository.CategoryRepository;
import eu.dnetlib.is.context.model.repository.ConceptLevel0Repository;
import eu.dnetlib.is.context.model.repository.ConceptLevel1Repository;
import eu.dnetlib.is.context.model.repository.ConceptLevel2Repository;
import eu.dnetlib.is.context.model.repository.ContextRepository;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.context.model.Category;
import eu.dnetlib.data.is.context.model.Context;
import eu.dnetlib.data.is.context.model.CtxChildInfo;
import eu.dnetlib.data.is.context.repository.CategoryRepository;
import eu.dnetlib.data.is.context.repository.ConceptLevel0Repository;
import eu.dnetlib.data.is.context.repository.ConceptLevel1Repository;
import eu.dnetlib.data.is.context.repository.ConceptLevel2Repository;
import eu.dnetlib.data.is.context.repository.ContextRepository;
import eu.dnetlib.is.errors.InformationServiceException;
@Service
public class ContextService {

View File

@ -0,0 +1,17 @@
package eu.dnetlib.is.errors;
public class DsmException extends Exception {
/**
*
*/
private static final long serialVersionUID = 8980196353591772127L;
public DsmException(final String message, final Throwable cause) {
super(message, cause);
}
public DsmException(final String message) {
super(message);
}
}

View File

@ -0,0 +1,15 @@
package eu.dnetlib.is.errors;
public class DsmForbiddenException extends DsmException {
private static final long serialVersionUID = 1283353531580924944L;
public DsmForbiddenException(final String message, final Throwable cause) {
super(message, cause);
}
public DsmForbiddenException(final String message) {
super(message);
}
}

View File

@ -0,0 +1,15 @@
package eu.dnetlib.is.errors;
public class DsmNotFoundException extends DsmException {
private static final long serialVersionUID = -1318946320754167152L;
public DsmNotFoundException(final String message, final Throwable cause) {
super(message, cause);
}
public DsmNotFoundException(final String message) {
super(message);
}
}

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.util;
package eu.dnetlib.is.errors;
public class InformationServiceException extends Exception {

View File

@ -11,19 +11,19 @@ import org.dom4j.Node;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import eu.dnetlib.is.context.model.Category;
import eu.dnetlib.is.context.model.ConceptLevel0;
import eu.dnetlib.is.context.model.ConceptLevel1;
import eu.dnetlib.is.context.model.ConceptLevel2;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.is.context.model.CtxChildInfo;
import eu.dnetlib.is.context.model.CtxInfo;
import eu.dnetlib.is.context.model.Parameter;
import eu.dnetlib.is.context.model.repository.CategoryRepository;
import eu.dnetlib.is.context.model.repository.ConceptLevel0Repository;
import eu.dnetlib.is.context.model.repository.ConceptLevel1Repository;
import eu.dnetlib.is.context.model.repository.ConceptLevel2Repository;
import eu.dnetlib.is.context.model.repository.ContextRepository;
import eu.dnetlib.data.is.context.model.Category;
import eu.dnetlib.data.is.context.model.ConceptLevel0;
import eu.dnetlib.data.is.context.model.ConceptLevel1;
import eu.dnetlib.data.is.context.model.ConceptLevel2;
import eu.dnetlib.data.is.context.model.Context;
import eu.dnetlib.data.is.context.model.CtxChildInfo;
import eu.dnetlib.data.is.context.model.CtxInfo;
import eu.dnetlib.data.is.context.model.Parameter;
import eu.dnetlib.data.is.context.repository.CategoryRepository;
import eu.dnetlib.data.is.context.repository.ConceptLevel0Repository;
import eu.dnetlib.data.is.context.repository.ConceptLevel1Repository;
import eu.dnetlib.data.is.context.repository.ConceptLevel2Repository;
import eu.dnetlib.data.is.context.repository.ContextRepository;
@Service
public class ContextImporter {

View File

@ -18,7 +18,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
@RestController
@RequestMapping("/api/import")

View File

@ -12,16 +12,16 @@ import org.dom4j.Node;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import eu.dnetlib.is.resource.model.SimpleResource;
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.resource.model.SimpleResource;
import eu.dnetlib.data.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.data.is.vocabulary.model.Synonym;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.data.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.data.is.vocabulary.repository.VocabularyTermRepository;
import eu.dnetlib.is.errors.InformationServiceException;
import eu.dnetlib.is.util.ResourceValidator;
import eu.dnetlib.is.util.XmlIndenter;
import eu.dnetlib.is.vocabulary.model.Synonym;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
@Service
public class OldProfilesImporter {

View File

@ -17,8 +17,8 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.is.wf.model.WfProcessExecution;
import eu.dnetlib.is.wf.repository.WfProcessExecutionRepository;
import eu.dnetlib.data.is.wf.model.WfProcessExecution;
import eu.dnetlib.data.is.wf.repository.WfProcessExecutionRepository;
@Service
public class WfHistoryImporter {

View File

@ -13,8 +13,8 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.resource.model.SimpleResource;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.resource.model.SimpleResource;
import eu.dnetlib.is.errors.InformationServiceException;
import eu.dnetlib.is.util.XmlIndenter;
public class AbstractResourceController extends AbstractDnetController {

View File

@ -15,8 +15,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.is.resource.model.SimpleResource;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.resource.model.SimpleResource;
import eu.dnetlib.is.errors.InformationServiceException;
@RestController
@RequestMapping("/ajax/resources")

View File

@ -9,8 +9,8 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.is.resource.model.ResourceType;
import eu.dnetlib.is.resource.repository.ResourceTypeRepository;
import eu.dnetlib.data.is.resource.model.ResourceType;
import eu.dnetlib.data.is.resource.repository.ResourceTypeRepository;
@RestController
@RequestMapping("/api/resources")

View File

@ -14,9 +14,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.PathVariable;
import eu.dnetlib.is.resource.model.SimpleResource;
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.data.is.resource.model.SimpleResource;
import eu.dnetlib.data.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.is.errors.InformationServiceException;
import eu.dnetlib.is.util.ResourceValidator;
@Service

View File

@ -23,8 +23,9 @@ import org.xml.sax.SAXException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.is.resource.model.ResourceType;
import eu.dnetlib.is.resource.repository.ResourceTypeRepository;
import eu.dnetlib.data.is.resource.model.ResourceType;
import eu.dnetlib.data.is.resource.repository.ResourceTypeRepository;
import eu.dnetlib.is.errors.InformationServiceException;
@Component
public class ResourceValidator {

View File

@ -7,8 +7,8 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.model.VocabularyTerm;
public class AbstractVocabularyController extends AbstractDnetController {

View File

@ -10,9 +10,9 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.is.errors.InformationServiceException;
@RestController
@RequestMapping("/ajax/vocs")

View File

@ -11,12 +11,12 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import eu.dnetlib.is.util.InformationServiceException;
import eu.dnetlib.is.vocabulary.model.Vocabulary;
import eu.dnetlib.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.is.vocabulary.model.VocabularyTermPK;
import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.is.vocabulary.model.VocabularyTerm;
import eu.dnetlib.data.is.vocabulary.model.VocabularyTermPK;
import eu.dnetlib.data.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.data.is.vocabulary.repository.VocabularyTermRepository;
import eu.dnetlib.is.errors.InformationServiceException;
@Service
public class VocabularyService {

View File

@ -12,8 +12,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.is.wf.model.WfProcessExecution;
import eu.dnetlib.is.wf.repository.WfProcessExecutionRepository;
import eu.dnetlib.data.is.wf.model.WfProcessExecution;
import eu.dnetlib.data.is.wf.repository.WfProcessExecutionRepository;
@RestController
@RequestMapping("/ajax/wfs")

View File

@ -0,0 +1,119 @@
package eu.dnetlib.openaire.dsm;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.errors.DsmException;
import eu.dnetlib.is.errors.DsmForbiddenException;
import eu.dnetlib.is.errors.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.Response;
/**
* Created by claudio on 18/07/2017.
*/
public abstract class AbstractDsmController extends AbstractDnetController {
private static final Log log = LogFactory.getLog(DsmException.class); // NOPMD by marko on 11/24/08 5:02 PM
@ResponseBody
@ExceptionHandler({
DsmException.class
})
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
public ErrorMessage handleDSMException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(DsmForbiddenException.class)
@ResponseStatus(value = HttpStatus.FORBIDDEN)
public ErrorMessage handleForbiddenException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler({
DsmNotFoundException.class
})
@ResponseStatus(value = HttpStatus.NOT_FOUND)
public ErrorMessage handleNotFoundException(final Exception e) {
return _handleError(e);
}
@ResponseBody
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseStatus(HttpStatus.BAD_REQUEST)
public List<ErrorMessage> processValidationError(final MethodArgumentNotValidException e) {
return e.getBindingResult()
.getFieldErrors()
.stream()
.map(fe -> new ErrorMessage(
String.format("field '%s'", fe.getField()),
String.format("rejected value '%s'", fe.getRejectedValue()),
fe.getDefaultMessage()))
.collect(Collectors.toList());
}
private ErrorMessage _handleError(final Exception e) {
log.error(e);
if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) {
return null; // socket is closed, cannot return any response
} else {
return new ErrorMessage(e);
}
}
// HELPERS
protected <T extends Response> T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) {
rsp.getHeader()
.setTime(stopWatch.getTime())
.setPage(page)
.setSize(size);
return rsp;
}
@JsonAutoDetect
public class ErrorMessage {
private final String message;
private final String details;
private final String stacktrace;
public ErrorMessage(final Exception e) {
this(e.getMessage(), "", ExceptionUtils.getStackTrace(e));
}
public ErrorMessage(final String message, final String details, final String stacktrace) {
this.message = message;
this.details = details;
this.stacktrace = stacktrace;
}
public String getMessage() {
return this.message;
}
public String getStacktrace() {
return this.stacktrace;
}
public String getDetails() {
return details;
}
}
}

View File

@ -0,0 +1,33 @@
package eu.dnetlib.openaire.dsm;
import java.util.List;
import eu.dnetlib.data.openaire.dsm.model.Api;
import eu.dnetlib.data.openaire.dsm.model.Datasource;
import eu.dnetlib.is.errors.DsmException;
import eu.dnetlib.is.errors.DsmForbiddenException;
import eu.dnetlib.is.errors.DsmNotFoundException;
public interface DatasourceManagerCommon {
Datasource getDs(String id) throws DsmException;
Datasource getDsByNsPrefix(String prefix) throws DsmException;
List<? extends Api> getApis(String dsId) throws DsmException;
void deleteDs(String dsId) throws DsmException;
void deleteApi(String dsId, String apiId) throws DsmForbiddenException, DsmNotFoundException;
void addApi(Api api) throws DsmException;
void setManaged(String id, boolean managed) throws DsmException;
boolean isManaged(String id) throws DsmException;
void saveDs(Datasource datasource) throws DsmException;
void updateCompliance(String dsId, String apiId, String compliance, boolean override) throws DsmException;
}

View File

@ -0,0 +1,281 @@
package eu.dnetlib.openaire.dsm;
import static eu.dnetlib.openaire.dsm.utils.DsmMappingUtils.asDbEntry;
import static eu.dnetlib.openaire.dsm.utils.DsmMappingUtils.copyNonNullProperties;
import static eu.dnetlib.openaire.dsm.utils.DsmMappingUtils.createId;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.validation.Valid;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.data.openaire.dsm.model.Api;
import eu.dnetlib.data.openaire.dsm.model.Datasource;
import eu.dnetlib.data.openaire.dsm.model.Identity;
import eu.dnetlib.is.errors.DsmException;
import eu.dnetlib.is.errors.DsmForbiddenException;
import eu.dnetlib.is.errors.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
import eu.dnetlib.openaire.dsm.domain.AggregationInfo;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
import eu.dnetlib.openaire.dsm.domain.Country;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.utils.ResponseUtils;
import eu.dnetlib.openaire.dsm.utils.WfLoggerClient;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/dsm/1.0")
@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
@Tag(name = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API")
public class DsmApiControllerV1 extends AbstractDsmController {
private static final Log log = LogFactory.getLog(DsmApiControllerV1.class);
@Autowired
private WfLoggerClient wfLoggerClient;
@Autowired
private DsmService dsmService;
@GetMapping("/ds/countries")
public List<Country> listCountries() throws DsmException {
return dsmService.listCountries();
}
@PostMapping("/ds/searchdetails/{page}/{size}")
public DatasourceDetailResponse searchDsDetails(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final DatasourceDetailResponse rsp = dsmService.searchDetails(requestSortBy, order, requestFilter, page, size);
return prepareResponse(page, size, stop, rsp);
}
@GetMapping("/ds/aggregationhistory/{dsId}")
public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final List<AggregationInfo> history = wfLoggerClient.getAggregationHistory(dsId);
final AggregationHistoryResponse rsp = new AggregationHistoryResponse(history);
rsp.setHeader(ResponseUtils.header(history.size()));
return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp);
}
@PostMapping("/ds/searchsnippet/{page}/{size}")
public DatasourceSnippetResponse searchSnippet(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final DatasourceSnippetResponse rsp = dsmService.searchSnippet(requestSortBy, order, requestFilter, page, size);
return prepareResponse(page, size, stop, rsp);
}
@PostMapping("/ds/searchregistered/{page}/{size}")
public DatasourceSnippetResponse searchRegistered(
@RequestParam final RequestSort requestSortBy,
@RequestParam final RequestSortOrder order,
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final Page<Datasource> dsPage = dsmService.searchRegistered(requestSortBy, order, requestFilter, page, size);
final DatasourceSnippetResponse rsp =
ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
return prepareResponse(page, size, stop, rsp);
}
@GetMapping("/ds/recentregistered/{size}")
public SimpleResponse<RegisteredDatasourceInfo> recentRegistered(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<RegisteredDatasourceInfo> rsp = dsmService.searchRecentRegistered(size);
return prepareResponse(1, size, stop, rsp);
}
@GetMapping("/ds/countregistered")
public Long countRegistered(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
return dsmService.countRegisteredAfter(fromDate, typologyFilter);
}
@GetMapping("/ds/api/{dsId}")
public ApiDetailsResponse getApi(
@PathVariable final String dsId) throws DsmException {
final StopWatch stop = StopWatch.createStarted();
final Datasource ds = dsmService.getDs(dsId);
final List<Api> apis = dsmService.getApis(dsId);
final List<ApiDetails> api = apis.stream()
.map(DsmMappingUtils::asDetails)
.map(a -> a.setEoscDatasourceType(ds.getEoscDatasourceType()))
.map(a -> a.setTypology(ds.getTypology()))
.collect(Collectors.toList());
final ApiDetailsResponse rsp = ResponseUtils.apiResponse(api, api.size());
return prepareResponse(0, rsp.getApi().size(), stop, rsp);
}
@PostMapping("/api/baseurl/{page}/{size}")
public List<String> searchBaseUrls(
@RequestBody final RequestFilter requestFilter,
@PathVariable final int page,
@PathVariable final int size) throws DsmException {
return dsmService.findApiBaseURLs(requestFilter, page, size);
}
@DeleteMapping("/ds/api/{apiId}")
public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException {
dsmService.deleteApi(null, apiId);
}
@PostMapping("/ds/manage")
public void setManaged(
@RequestParam final String id,
@RequestParam final boolean managed) throws DsmException {
log.info(String.format("updated ds '%s' managed with '%s'", id, managed));
dsmService.setManaged(id, managed);
}
@GetMapping("/ds/managed/{id}")
public boolean isManaged(@PathVariable final String id) throws DsmException {
return dsmService.isManaged(id);
}
@PostMapping("/ds/add")
public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException {
if (dsmService.existDs(datasource.getId())) { // TODO further check that the DS doesn't have any API
throw new DsmException(String.format("cannot register, datasource already defined '%s'", datasource.getId()));
}
dsmService.saveDs(asDbEntry(datasource));
log.info("DS saved, " + datasource.getId());
}
@PostMapping("/ds/addWithApis")
public void saveDsWithApis(@Valid @RequestBody final DatasourceDetailsWithApis d) throws DsmException {
if (d.getDatasource() == null) { throw new DsmException("Datasource field is null"); }
if (dsmService.existDs(d.getDatasource().getId())) { // TODO further check that the DS doesn't have any API
throw new DsmException(String.format("cannot register, datasource already defined '%s'", d.getDatasource().getId()));
}
dsmService.addDsAndApis(asDbEntry(d.getDatasource()), d.getApis());
}
@PostMapping("/ds/update")
public void updateDatasource(@RequestBody final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException {
// initialize with current values from DB
final Datasource ds = dsmService.getDs(d.getId());
if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); }
final Datasource update = asDbEntry(d);
if (ds.getIdentities() != null) {
final Set<Identity> identities = new HashSet<>(
Stream.of(update.getIdentities(), ds.getIdentities())
.flatMap(Collection::stream)
.collect(Collectors.toMap(i -> i.getIssuertype() + i.getPid(), Function.identity(), (i1, i2) -> i1))
.values());
copyNonNullProperties(update, ds);
ds.setIdentities(identities);
} else {
copyNonNullProperties(update, ds);
}
dsmService.saveDs(ds);
}
@PostMapping("/ds/api/baseurl")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void updateBaseUrl(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String baseUrl) throws DsmException {
log.info(String.format("updated api '%s' baseurl with '%s'", apiId, baseUrl));
dsmService.updateApiBaseUrl(apiId, baseUrl);
}
@PostMapping("/ds/api/compliance")
public void updateCompliance(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String compliance,
@RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException {
log.info(String.format("updated api '%s' compliance with '%s'", apiId, compliance));
dsmService.updateCompliance(null, apiId, compliance, override);
}
@PostMapping("/ds/api/oaiset")
public void updateOaiSetl(
@RequestParam final String dsId,
@RequestParam final String apiId,
@RequestParam final String oaiSet) throws DsmException, DsmNotFoundException {
dsmService.upsertApiOaiSet(apiId, oaiSet);
}
@PostMapping("/ds/api/add")
public void addApi(@RequestBody final ApiDetails api) throws DsmException {
if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException("missing datasource id"); }
if (StringUtils.isBlank(api.getId())) {
api.setId(createId(api));
log.info(String.format("missing api id, created '%s'", api.getId()));
}
dsmService.addApi(asDbEntry(api));
log.info("API saved, id: " + api.getId());
}
}

View File

@ -0,0 +1,51 @@
package eu.dnetlib.openaire.dsm;
import java.util.List;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.utils.ResponseUtils;
import io.swagger.v3.oas.annotations.tags.Tag;
@RestController
@RequestMapping("/api/dsm/2.0")
@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
@Tag(name = "OpenAIRE DSM API (version 2.0)", description = "the OpenAIRE Datasource Manager API 2.0")
public class DsmApiControllerV2 extends AbstractDsmController {
@Autowired
private DsmService dsmService;
@GetMapping("/recentregistered/{size}")
public SimpleResponse<SimpleDatasourceInfo> recentRegisteredV2(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<SimpleDatasourceInfo> rsp = dsmService.searchRecentRegisteredV2(size);
return prepareResponse(1, size, stop, rsp);
}
@GetMapping("/countfirstcollect")
public Long countFirstCollectAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
return dsmService.countFirstCollect(fromDate, typologyFilter);
}
@GetMapping("/firstCollected")
public SimpleResponse<SimpleDatasourceInfo> firstCollectedAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final List<SimpleDatasourceInfo> list = dsmService.getFirstCollectedAfter(fromDate, typologyFilter);
final SimpleResponse<SimpleDatasourceInfo> rsp = ResponseUtils.simpleResponse(list);
return prepareResponse(1, list.size(), stop, rsp);
}
}

View File

@ -0,0 +1,448 @@
package eu.dnetlib.openaire.dsm;
import static eu.dnetlib.openaire.dsm.utils.DatasourceSpecs.apiSpec;
import static eu.dnetlib.openaire.dsm.utils.DatasourceSpecs.dsRegisteredbyNotNullSpec;
import static eu.dnetlib.openaire.dsm.utils.DatasourceSpecs.dsSpec;
import static eu.dnetlib.openaire.dsm.utils.DsmMappingUtils.asDbEntry;
import static eu.dnetlib.openaire.dsm.utils.DsmMappingUtils.createId;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.EntityNotFoundException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Lists;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
import eu.dnetlib.data.openaire.dsm.model.Api;
import eu.dnetlib.data.openaire.dsm.model.ApiParam;
import eu.dnetlib.data.openaire.dsm.model.Datasource;
import eu.dnetlib.data.openaire.dsm.model.DatasourceApi;
import eu.dnetlib.data.openaire.dsm.repository.ApiRepository;
import eu.dnetlib.data.openaire.dsm.repository.CountryTermRepository;
import eu.dnetlib.data.openaire.dsm.repository.DatasourceApiRepository;
import eu.dnetlib.data.openaire.dsm.repository.DatasourceRepository;
import eu.dnetlib.is.errors.DsmException;
import eu.dnetlib.is.errors.DsmForbiddenException;
import eu.dnetlib.is.errors.DsmNotFoundException;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.Country;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.utils.ResponseUtils;
import eu.dnetlib.openaire.dsm.utils.VocabularyClient;
/**
* Created by claudio on 20/10/2016.
*/
@Service
@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
public class DsmService {
private static final Log log = LogFactory.getLog(DsmService.class);
public static final String OAI = "oai";
public static final String SET = "set";
@Autowired
private CountryTermRepository countryTermRepository;
@Autowired
private DatasourceRepository dsRepository;
@Autowired
private ApiRepository apiRepository;
@Autowired
private DatasourceApiRepository dsApiRepository;
@Autowired
private VocabularyClient vocabularyClient;
@Autowired
private JdbcTemplate jdbcTemplate;
public List<Country> listCountries() throws DsmException {
final List<Country> countries = Lists.newArrayList();
// TODO Usare solo vocabolari???
final Vocabulary v = vocabularyClient.getCountries();
countries.addAll(countryTermRepository.findAll()
.stream()
.filter(Objects::nonNull)
.map(t -> new Country(t.getTerm(), t.getName()))
.collect(Collectors.toList()));
return countries;
}
@Transactional
public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size) throws DsmException {
final Page<Datasource> dsPage = search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
}
public DatasourceDetailResponse searchDetails(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size) throws DsmException {
final Page<Datasource> dsPage = search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.detailsResponse(dsPage.map(DsmMappingUtils::asDetails).getContent(), dsPage.getTotalElements());
}
private Page<Datasource> search(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<Datasource> spec = dsSpec(requestSortBy, order, requestFilter);
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
public Page<Datasource> searchRegistered(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<Datasource> spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec());
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
public Datasource getDs(final String dsId) throws DsmException {
return dsRepository.findById(dsId).orElseThrow(() -> new DsmException("Datasource not found. ID: " + dsId));
}
public Datasource getDsByNsPrefix(final String prefix) throws DsmException {
return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix));
}
public void setManaged(final String id, final boolean managed) {
log.info(String.format("setting managed = '%s' for ds '%s'", managed, id));
dsRepository.setManaged(id, managed);
apiRepository.setRemovable(id, true);
}
public boolean isManaged(final String id) {
return dsRepository.isManaged(id);
}
public void updateCompliance(final String dsId, final String apiId, final String compliance, final boolean override) {
log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId));
apiRepository.updateCompatibility(apiId, compliance);
}
public List<Api> getApis(final String dsId) {
return apiRepository.findByDatasource(dsId);
}
public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException {
final Api api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
try {
if (!api.getRemovable()) { throw new DsmForbiddenException("api is not removable"); }
apiRepository.deleteById(apiId);
log.info(String.format("deleted api '%s'", apiId));
} catch (final EntityNotFoundException e) {
throw new DsmNotFoundException("api not found");
}
}
public void addApi(final Api api) {
apiRepository.save(api);
}
public boolean existDs(final String dsId) throws DsmException {
return dsRepository.existsById(dsId);
}
public void saveDs(final Datasource d) {
log.info(String.format("saving datasource '%s'", d.getId()));
final Datasource datasource = dsRepository.save(d);
log.info(String.format("saved datasource '%s'", datasource.getId()));
ensureRegistrationDate(d.getId());
}
public void deleteDs(final String dsId) {
dsRepository.deleteById(dsId);
log.info(String.format("deleted datasource '%s'", dsId));
}
public void updateName(final String dsId, final String officialname, final String englishname) {
// TODO what if one of the two names is null or empty?
dsRepository.setDatasourcename(dsId, officialname, englishname);
}
public void updateLogoUrl(final String dsId, final String logourl) throws DsmException {
dsRepository.setLogoUrl(dsId, logourl);
}
public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) {
dsRepository.setCoordinates(dsId, latitude, longitude);
}
public void updateApiBaseUrl(final String apiId, final String baseurl) {
apiRepository.setBaseurl(apiId, baseurl);
}
@Transactional
public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException, DsmNotFoundException {
final Api api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
if (OAI.equalsIgnoreCase(api.getProtocol())) {
final Set<ApiParam> apiParams = api.getApiParams();
if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) {
apiRepository.addApiParam(apiId, SET, oaiSet);
log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet));
return true;
} else {
apiRepository.updateOaiSet(apiId, oaiSet);
log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet));
return false;
}
} else {
throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol()));
}
}
public List<String> findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
final PageRequest pageable = PageRequest.of(page, size);
final Specification<DatasourceApi> spec = apiSpec(requestFilter);
final Set<String> set = dsApiRepository.findAll(spec, pageable)
.getContent()
.stream()
.map(DatasourceApi::getBaseurl)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toCollection(HashSet::new));
return Lists.newArrayList(set);
}
public void updateTimezone(final String dsId, final String timezone) {
dsRepository.setTimezone(dsId, timezone);
}
public void updateEoscDatasourceType(final String dsId, final String type) throws DsmException {
if (vocabularyClient.isValidTerm("eosc_datasource_types", type)) {
throw new DsmException(String.format("invalid datasource type '%s', provide one according to vocabulary eosc_datasource_types"));
}
dsRepository.setEoscDatasourceType(dsId, type);
}
public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException {
ensureRegistrationDate(dsId);
dsRepository.setRegisteringUser(dsId, registeredBy);
}
public void updatePlatform(final String dsId, final String platform) throws DsmException {
dsRepository.setPlatform(dsId, platform);
}
public SimpleResponse<RegisteredDatasourceInfo> searchRecentRegistered(final int size) throws DsmException {
try {
final List<RegisteredDatasourceInfo> list =
querySql("recent_registered_datasources.sql.st", BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%");
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public SimpleResponse<SimpleDatasourceInfo> searchRecentRegisteredV2(final int size) throws DsmException {
try {
final List<SimpleDatasourceInfo> list = querySql("recent_registered_datasources_v2.sql.st", rowMapperForSimpleDatasourceInfo(), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public Long countFirstCollect(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
return querySql("count_first_collected_datasources_fromDate_typology.st.sql", Long.class, typeFilter + "%", fromDate);
} else {
return querySql("count_first_collected_datasources_fromDate.st.sql", Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw new DsmException("error searching datasources using the first collection date", e);
}
}
public List<SimpleDatasourceInfo> getFirstCollectedAfter(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
return querySql("first_collected_datasources_fromDate_typology.st.sql", rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate);
} else {
return querySql("first_collected_datasources_fromDate.st.sql", rowMapperForSimpleDatasourceInfo(), fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw new DsmException("error searching datasources using the first collection date", e);
}
}
private <T> List<T> querySql(final String sqlFile, final RowMapper<T> rowMapper, final Object... params) throws IOException {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/dsm/" + sqlFile), StandardCharsets.UTF_8);
return jdbcTemplate.query(sql, rowMapper, params);
}
private <T> T querySql(final String sqlFile, final Class<T> clazz, final Object... params) throws IOException {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/dsm/" + sqlFile), StandardCharsets.UTF_8);
return jdbcTemplate.queryForObject(sql, clazz, params);
}
// HELPER
private void ensureRegistrationDate(final String dsId) {
if (!dsRepository.hasRegistrationdate(dsId)) {
log.info("setting registration date for datasource: " + dsId);
dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis()));
}
}
private RowMapper<SimpleDatasourceInfo> rowMapperForSimpleDatasourceInfo() {
return (rs, rowNum) -> {
final SimpleDatasourceInfo info = new SimpleDatasourceInfo();
info.setId(rs.getString("id"));
info.setOfficialName(rs.getString("officialName"));
info.setEnglishName(rs.getString("englishName"));
info.setTypology(rs.getString("typology"));
info.setEoscType(rs.getString("eoscType"));
info.setEoscDatasourceType(rs.getString("eoscDatasourceType"));
info.setRegisteredBy(rs.getString("registeredBy"));
info.setRegistrationDate(rs.getString("registrationDate"));
info.setFirstCollectionDate(rs.getString("firstCollectionDate"));
info.setLastCollectionDate(rs.getString("lastCollectionDate"));
info.setLastCollectionTotal(rs.getLong("lastCollectionTotal"));
final Set<String> compatibilities = new HashSet<>();
for (final String s : (String[]) rs.getArray("compatibilities").getArray()) {
compatibilities.add(s);
}
// The order of the condition is important
if (compatibilities.contains("openaire-cris_1.1")) {
info.setCompatibility("openaire-cris_1.1");
} else if (compatibilities.contains("openaire4.0")) {
info.setCompatibility("openaire4.0");
} else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) {
info.setCompatibility("driver-openaire2.0");
} else if (compatibilities.contains("driver")) {
info.setCompatibility("driver");
} else if (compatibilities.contains("openaire2.0")) {
info.setCompatibility("openaire2.0");
} else if (compatibilities.contains("openaire3.0")) {
info.setCompatibility("openaire3.0");
} else if (compatibilities.contains("openaire2.0_data")) {
info.setCompatibility("openaire2.0_data");
} else if (compatibilities.contains("native")) {
info.setCompatibility("native");
} else if (compatibilities.contains("hostedBy")) {
info.setCompatibility("hostedBy");
} else if (compatibilities.contains("notCompatible")) {
info.setCompatibility("notCompatible");
} else {
info.setCompatibility("UNKNOWN");
}
for (final String s : (String[]) rs.getArray("organizations").getArray()) {
info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim());
}
return info;
};
}
@Transactional
public void addDsAndApis(final Datasource ds, final List<ApiDetails> apis) throws DsmException {
try {
saveDs(ds);
if (apis != null) {
for (final ApiDetails api : apis) {
api.setDatasource(ds.getId());
if (StringUtils.isBlank(api.getId())) {
api.setId(createId(api));
log.info(String.format("missing api id, created '%s'", api.getId()));
}
addApi(asDbEntry(api));
log.info("API saved, id: " + api.getId());
}
}
} catch (final Throwable e) {
log.error("Error saving ds and/or api", e);
throw new DsmException("Error saving ds and/or api", e);
}
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class AggregationHistoryResponse extends Response {
private List<AggregationInfo> aggregationInfo;
public AggregationHistoryResponse(final List<AggregationInfo> aggregationInfo) {
super();
this.aggregationInfo = aggregationInfo;
}
public List<AggregationInfo> getAggregationInfo() {
return aggregationInfo;
}
public void setAggregationInfo(final List<AggregationInfo> aggregationInfo) {
this.aggregationInfo = aggregationInfo;
}
}

View File

@ -0,0 +1,56 @@
package eu.dnetlib.openaire.dsm.domain;
public abstract class AggregationInfo {
private int numberOfRecords;
private String date;
private AggregationStage aggregationStage;
private boolean indexedVersion = false;
private boolean completedSuccessfully = true;
public AggregationInfo() {}
public int getNumberOfRecords() {
return numberOfRecords;
}
public void setNumberOfRecords(final int numberOfRecords) {
this.numberOfRecords = numberOfRecords;
}
public String getDate() {
return date;
}
public void setDate(final String date) {
this.date = date;
}
public AggregationStage getAggregationStage() {
return aggregationStage;
}
public void setAggregationStage(final AggregationStage aggregationStage) {
this.aggregationStage = aggregationStage;
}
public boolean isIndexedVersion() {
return indexedVersion;
}
public void setIndexedVersion(final boolean indexedVersion) {
this.indexedVersion = indexedVersion;
}
public boolean isCompletedSuccessfully() {
return completedSuccessfully;
}
public void setCompletedSuccessfully(final boolean completedSuccessfully) {
this.completedSuccessfully = completedSuccessfully;
}
}

View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.dsm.domain;
public enum AggregationStage {
COLLECT,
TRANSFORM;
public static AggregationStage parse(final String s) {
switch (s) {
case "collect":
case "collection":
case "COLLECT":
case "COLLECTION":
return AggregationStage.COLLECT;
case "transform":
case "transformation":
case "TRANSFORM":
case "TRANSFORMATION":
case "transformDatasets":
case "transformPublications":
return AggregationStage.TRANSFORM;
}
throw new IllegalArgumentException("invalid AggregationStage: " + s);
}
}

View File

@ -0,0 +1,210 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Api model", description = "provides information about the datasource API")
public class ApiDetails extends ApiIgnoredProperties {
private String id = null;
private String protocol = null;
private String datasource = null;
private String contentdescription = null;
private String eoscDatasourceType = null;
private String compatibility;
private String compatibilityOverride;
private Integer lastCollectionTotal;
private Date lastCollectionDate;
private Integer lastAggregationTotal;
private Date lastAggregationDate;
private Integer lastDownloadTotal;
private Date lastDownloadDate;
private String baseurl;
protected Boolean removable = false;
private Set<ApiParamDetails> apiParams;
private String metadataIdentifierPath = "";
private String typology = null;
public String getId() {
return id;
}
public String getProtocol() {
return protocol;
}
public String getDatasource() {
return datasource;
}
public String getContentdescription() {
return contentdescription;
}
public String getCompatibility() {
return compatibility;
}
public Integer getLastCollectionTotal() {
return lastCollectionTotal;
}
public Date getLastCollectionDate() {
return lastCollectionDate;
}
public Integer getLastAggregationTotal() {
return lastAggregationTotal;
}
public Date getLastAggregationDate() {
return lastAggregationDate;
}
public Integer getLastDownloadTotal() {
return lastDownloadTotal;
}
public Date getLastDownloadDate() {
return lastDownloadDate;
}
public String getBaseurl() {
return baseurl;
}
public ApiDetails setId(final String id) {
this.id = id;
return this;
}
public ApiDetails setProtocol(final String protocol) {
this.protocol = protocol;
return this;
}
public ApiDetails setDatasource(final String datasource) {
this.datasource = datasource;
return this;
}
public ApiDetails setContentdescription(final String contentdescription) {
this.contentdescription = contentdescription;
return this;
}
public ApiDetails setCompatibility(final String compatibility) {
this.compatibility = compatibility;
return this;
}
public ApiDetails setLastCollectionTotal(final Integer lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
return this;
}
public ApiDetails setLastCollectionDate(final Date lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
return this;
}
public ApiDetails setLastAggregationTotal(final Integer lastAggregationTotal) {
this.lastAggregationTotal = lastAggregationTotal;
return this;
}
public ApiDetails setLastAggregationDate(final Date lastAggregationDate) {
this.lastAggregationDate = lastAggregationDate;
return this;
}
public ApiDetails setLastDownloadTotal(final Integer lastDownloadTotal) {
this.lastDownloadTotal = lastDownloadTotal;
return this;
}
public ApiDetails setLastDownloadDate(final Date lastDownloadDate) {
this.lastDownloadDate = lastDownloadDate;
return this;
}
public ApiDetails setBaseurl(final String baseurl) {
this.baseurl = baseurl;
return this;
}
public Set<ApiParamDetails> getApiParams() {
return apiParams;
}
public void setApiParams(final Set<ApiParamDetails> apiParams) {
this.apiParams = apiParams;
}
public String getCompatibilityOverride() {
return compatibilityOverride;
}
public ApiDetails setCompatibilityOverride(final String compatibilityOverride) {
this.compatibilityOverride = compatibilityOverride;
return this;
}
public Boolean getRemovable() {
return removable;
}
public ApiDetails setRemovable(final Boolean removable) {
this.removable = removable;
return this;
}
public String getMetadataIdentifierPath() {
return metadataIdentifierPath;
}
public ApiDetails setMetadataIdentifierPath(final String metadataIdentifierPath) {
this.metadataIdentifierPath = metadataIdentifierPath;
return this;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public ApiDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
public String getTypology() {
return typology;
}
public ApiDetails setTypology(final String typology) {
this.typology = typology;
return this;
}
}

View File

@ -0,0 +1,23 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
public class ApiDetailsResponse extends Response {
private List<ApiDetails> api;
public List<ApiDetails> getApi() {
return api;
}
public ApiDetailsResponse setApi(final List<ApiDetails> api) {
this.api = api;
return this;
}
}

View File

@ -0,0 +1,72 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
public abstract class ApiIgnoredProperties {
@JsonIgnore
protected Boolean active = false;
@JsonIgnore
protected String lastCollectionMdid;
@JsonIgnore
protected String lastAggregationMdid;
@JsonIgnore
protected String lastDownloadObjid;
@JsonIgnore
protected String lastValidationJob;
@JsonIgnore
protected boolean compatibilityOverrided;
public Boolean getActive() {
return active;
}
public void setActive(final Boolean active) {
this.active = active;
}
public String getLastCollectionMdid() {
return lastCollectionMdid;
}
public void setLastCollectionMdid(final String lastCollectionMdid) {
this.lastCollectionMdid = lastCollectionMdid;
}
public String getLastAggregationMdid() {
return lastAggregationMdid;
}
public void setLastAggregationMdid(final String lastAggregationMdid) {
this.lastAggregationMdid = lastAggregationMdid;
}
public String getLastDownloadObjid() {
return lastDownloadObjid;
}
public void setLastDownloadObjid(final String lastDownloadObjid) {
this.lastDownloadObjid = lastDownloadObjid;
}
public String getLastValidationJob() {
return lastValidationJob;
}
public void setLastValidationJob(final String lastValidationJob) {
this.lastValidationJob = lastValidationJob;
}
public boolean isCompatibilityOverrided() {
return compatibilityOverrided;
}
public void setCompatibilityOverrided(final boolean compatibilityOverrided) {
this.compatibilityOverrided = compatibilityOverrided;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
public class ApiParamDetails {
protected String param;
protected String value;
public String getParam() {
return param;
}
public void setParam(final String param) {
this.param = param;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
}

View File

@ -0,0 +1,21 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
public class CollectionInfo extends AggregationInfo {
private CollectionMode collectionMode;
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

View File

@ -0,0 +1,12 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
public enum CollectionMode {
REFRESH,
INCREMENTAL
}

View File

@ -0,0 +1,31 @@
package eu.dnetlib.openaire.dsm.domain;
public class Country {
private String code;
private String name;
public Country() {}
public Country(final String code, final String name) {
this.code = code;
this.name = name;
}
public String getCode() {
return code;
}
public void setCode(final String code) {
this.code = code;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class DatasourceDetailResponse extends Response {
private List<DatasourceDetails> datasourceInfo;
public DatasourceDetailResponse(final List<DatasourceDetails> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceDetails> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(final List<DatasourceDetails> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

View File

@ -0,0 +1,417 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import javax.persistence.Transient;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
@Schema(name = "Datasource model", description = "provides information about the datasource")
public class DatasourceDetails extends DatasourceIgnoredProperties {
@NotBlank
private String id;
@Transient
private String openaireId;
@NotBlank
private String officialname;
@NotBlank
private String englishname;
private String websiteurl;
private String logourl;
@Email
private String contactemail;
private Double latitude;
private Double longitude;
private String timezone;
@NotBlank
private String namespaceprefix;
private String languages;
private Date dateofvalidation;
@NotBlank
private String eoscDatasourceType;
private Date dateofcollection;
private String platform;
private String activationId;
private String description;
private String issn;
private String eissn;
private String lissn;
@Email
private String registeredby;
private String subjects;
protected String aggregator = "OPENAIRE";
protected String collectedfrom;
private Boolean managed;
private Boolean consentTermsOfUse;
private Boolean fullTextDownload;
private Date consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate;
private Set<OrganizationDetails> organizations;
private Set<IdentitiesDetails> identities;
private String status;
@Deprecated
private String typology;
private Date registrationdate;
public String getId() {
return id;
}
public String getOpenaireId() {
return openaireId;
}
public String getOfficialname() {
return officialname;
}
public String getEnglishname() {
return englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getContactemail() {
return contactemail;
}
public Double getLatitude() {
return latitude;
}
public Double getLongitude() {
return longitude;
}
public String getTimezone() {
return timezone;
}
public String getLanguages() {
return languages;
}
public String getNamespaceprefix() {
return namespaceprefix;
}
public Date getDateofvalidation() {
return dateofvalidation;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public Date getDateofcollection() {
return dateofcollection;
}
public String getPlatform() {
return platform;
}
public String getActivationId() {
return activationId;
}
public String getDescription() {
return description;
}
public String getIssn() {
return issn;
}
public String getEissn() {
return eissn;
}
public String getLissn() {
return lissn;
}
public String getRegisteredby() {
return registeredby;
}
public String getSubjects() {
return subjects;
}
public String getAggregator() {
return aggregator;
}
public String getCollectedfrom() {
return collectedfrom;
}
public Boolean getManaged() {
return managed;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public Set<OrganizationDetails> getOrganizations() {
return organizations;
}
public Set<IdentitiesDetails> getIdentities() {
return identities;
}
public DatasourceDetails setId(final String id) {
this.id = id;
return this;
}
public DatasourceDetails setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
return this;
}
public DatasourceDetails setOfficialname(final String officialname) {
this.officialname = officialname;
return this;
}
public DatasourceDetails setEnglishname(final String englishname) {
this.englishname = englishname;
return this;
}
public DatasourceDetails setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public DatasourceDetails setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public DatasourceDetails setContactemail(final String contactemail) {
this.contactemail = contactemail;
return this;
}
public DatasourceDetails setLatitude(final Double latitude) {
this.latitude = latitude;
return this;
}
public DatasourceDetails setLongitude(final Double longitude) {
this.longitude = longitude;
return this;
}
public DatasourceDetails setTimezone(final String timezone) {
this.timezone = timezone;
return this;
}
public DatasourceDetails setLanguages(final String languages) {
this.languages = languages;
return this;
}
public DatasourceDetails setNamespaceprefix(final String namespaceprefix) {
this.namespaceprefix = namespaceprefix;
return this;
}
public DatasourceDetails setDateofvalidation(final Date dateofvalidation) {
this.dateofvalidation = dateofvalidation;
return this;
}
public DatasourceDetails setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
public DatasourceDetails setDateofcollection(final Date dateofcollection) {
this.dateofcollection = dateofcollection;
return this;
}
public DatasourceDetails setPlatform(final String platform) {
this.platform = platform;
return this;
}
public DatasourceDetails setActivationId(final String activationId) {
this.activationId = activationId;
return this;
}
public DatasourceDetails setDescription(final String description) {
this.description = description;
return this;
}
public DatasourceDetails setIssn(final String issn) {
this.issn = issn;
return this;
}
public DatasourceDetails setEissn(final String eissn) {
this.eissn = eissn;
return this;
}
public DatasourceDetails setLissn(final String lissn) {
this.lissn = lissn;
return this;
}
public DatasourceDetails setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
return this;
}
public DatasourceDetails setSubjects(final String subjects) {
this.subjects = subjects;
return this;
}
public DatasourceDetails setAggregator(final String aggregator) {
this.aggregator = aggregator;
return this;
}
public DatasourceDetails setCollectedfrom(final String collectedfrom) {
this.collectedfrom = collectedfrom;
return this;
}
public DatasourceDetails setManaged(final Boolean managed) {
this.managed = managed;
return this;
}
public DatasourceDetails setOrganizations(final Set<OrganizationDetails> organizations) {
this.organizations = organizations;
return this;
}
public DatasourceDetails setIdentities(final Set<IdentitiesDetails> identities) {
this.identities = identities;
return this;
}
public DatasourceDetails setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public DatasourceDetails setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public String getStatus() {
return status;
}
public DatasourceDetails setStatus(final String status) {
this.status = status;
return this;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public DatasourceDetails setTypology(final String typology) {
this.typology = typology;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
public Date getRegistrationdate() {
return registrationdate;
}
public DatasourceDetails setRegistrationdate(final Date registrationdate) {
this.registrationdate = registrationdate;
return this;
}
}

View File

@ -0,0 +1,248 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import java.util.Set;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
@Schema(name = "Datasource updatable fields model", description = "provides information about the datasource field that can be updated")
public class DatasourceDetailsUpdate {
@NotBlank
private String id;
@NotBlank
private String officialname;
@NotBlank
private String englishname;
private String websiteurl;
private String logourl;
@Email
private String contactemail;
private Double latitude;
private Double longitude;
private String timezone;
@Deprecated
private String typology;
private String eoscDatasourceType;
private String platform;
private String description;
@Email
private String registeredby;
private Boolean managed;
private Set<IdentitiesDetails> identities;
private Boolean consentTermsOfUse;
private Date consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate;
private Boolean fullTextDownload;
public String getId() {
return id;
}
public String getOfficialname() {
return officialname;
}
public String getEnglishname() {
return englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getContactemail() {
return contactemail;
}
public Double getLatitude() {
return latitude;
}
public Double getLongitude() {
return longitude;
}
public String getTimezone() {
return timezone;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public String getPlatform() {
return platform;
}
public String getDescription() {
return description;
}
public String getRegisteredby() {
return registeredby;
}
public Boolean getManaged() {
return managed;
}
public Set<IdentitiesDetails> getIdentities() {
return identities;
}
public DatasourceDetailsUpdate setId(final String id) {
this.id = id;
return this;
}
public DatasourceDetailsUpdate setOfficialname(final String officialname) {
this.officialname = officialname;
return this;
}
public DatasourceDetailsUpdate setEnglishname(final String englishname) {
this.englishname = englishname;
return this;
}
public DatasourceDetailsUpdate setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public DatasourceDetailsUpdate setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public DatasourceDetailsUpdate setContactemail(final String contactemail) {
this.contactemail = contactemail;
return this;
}
public DatasourceDetailsUpdate setLatitude(final Double latitude) {
this.latitude = latitude;
return this;
}
public DatasourceDetailsUpdate setLongitude(final Double longitude) {
this.longitude = longitude;
return this;
}
public DatasourceDetailsUpdate setTimezone(final String timezone) {
this.timezone = timezone;
return this;
}
public DatasourceDetailsUpdate setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
return this;
}
public DatasourceDetailsUpdate setPlatform(final String platform) {
this.platform = platform;
return this;
}
public DatasourceDetailsUpdate setDescription(final String description) {
this.description = description;
return this;
}
public DatasourceDetailsUpdate setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
return this;
}
public DatasourceDetailsUpdate setManaged(final Boolean managed) {
this.managed = managed;
return this;
}
public DatasourceDetailsUpdate setIdentities(final Set<IdentitiesDetails> identities) {
this.identities = identities;
return this;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public DatasourceDetailsUpdate setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public DatasourceDetailsUpdate setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public DatasourceDetailsUpdate setTypology(final String typology) {
this.typology = typology;
return this;
}
}

View File

@ -0,0 +1,37 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
/**
* Created by claudio on 12/09/16.
*/
@JsonAutoDetect
@Schema(name = "Datasource model with apis", description = "provides information about the datasource and its apis")
public class DatasourceDetailsWithApis {
private DatasourceDetails datasource;
private List<ApiDetails> apis = new ArrayList<>();
public DatasourceDetails getDatasource() {
return datasource;
}
public void setDatasource(final DatasourceDetails datasource) {
this.datasource = datasource;
}
public List<ApiDetails> getApis() {
return apis;
}
public void setApis(final List<ApiDetails> apis) {
this.apis = apis;
}
}

View File

@ -0,0 +1,207 @@
package eu.dnetlib.openaire.dsm.domain;
import java.sql.Date;
import com.fasterxml.jackson.annotation.JsonIgnore;
public abstract class DatasourceIgnoredProperties {
@JsonIgnore
protected String od_contenttypes;
@JsonIgnore
protected String provenanceaction;
@JsonIgnore
protected Date releasestartdate;
@JsonIgnore
protected Date releaseenddate;
@JsonIgnore
protected String missionstatementurl;
@JsonIgnore
protected Boolean dataprovider;
@JsonIgnore
protected Boolean serviceprovider;
@JsonIgnore
protected String databaseaccesstype;
@JsonIgnore
protected String datauploadtype;
@JsonIgnore
protected String databaseaccessrestriction;
@JsonIgnore
protected String datauploadrestriction;
@JsonIgnore
protected Boolean versioning;
@JsonIgnore
protected String citationguidelineurl;
@JsonIgnore
protected String qualitymanagementkind;
@JsonIgnore
protected String pidsystems;
@JsonIgnore
protected String certificates;
@JsonIgnore
protected String eoscType;
@JsonIgnore
protected Boolean dedupMainService;
public String getOd_contenttypes() {
return od_contenttypes;
}
public void setOd_contenttypes(final String od_contenttypes) {
this.od_contenttypes = od_contenttypes;
}
public String getProvenanceaction() {
return provenanceaction;
}
public void setProvenanceaction(final String provenanceaction) {
this.provenanceaction = provenanceaction;
}
public Date getReleasestartdate() {
return releasestartdate;
}
public void setReleasestartdate(final Date releasestartdate) {
this.releasestartdate = releasestartdate;
}
public Date getReleaseenddate() {
return releaseenddate;
}
public void setReleaseenddate(final Date releaseenddate) {
this.releaseenddate = releaseenddate;
}
public String getMissionstatementurl() {
return missionstatementurl;
}
public void setMissionstatementurl(final String missionstatementurl) {
this.missionstatementurl = missionstatementurl;
}
public Boolean getDataprovider() {
return dataprovider;
}
public void setDataprovider(final Boolean dataprovider) {
this.dataprovider = dataprovider;
}
public Boolean getServiceprovider() {
return serviceprovider;
}
public void setServiceprovider(final Boolean serviceprovider) {
this.serviceprovider = serviceprovider;
}
public String getDatabaseaccesstype() {
return databaseaccesstype;
}
public void setDatabaseaccesstype(final String databaseaccesstype) {
this.databaseaccesstype = databaseaccesstype;
}
public String getDatauploadtype() {
return datauploadtype;
}
public void setDatauploadtype(final String datauploadtype) {
this.datauploadtype = datauploadtype;
}
public String getDatabaseaccessrestriction() {
return databaseaccessrestriction;
}
public void setDatabaseaccessrestriction(final String databaseaccessrestriction) {
this.databaseaccessrestriction = databaseaccessrestriction;
}
public String getDatauploadrestriction() {
return datauploadrestriction;
}
public void setDatauploadrestriction(final String datauploadrestriction) {
this.datauploadrestriction = datauploadrestriction;
}
public Boolean getVersioning() {
return versioning;
}
public void setVersioning(final Boolean versioning) {
this.versioning = versioning;
}
public String getCitationguidelineurl() {
return citationguidelineurl;
}
public void setCitationguidelineurl(final String citationguidelineurl) {
this.citationguidelineurl = citationguidelineurl;
}
public String getQualitymanagementkind() {
return qualitymanagementkind;
}
public void setQualitymanagementkind(final String qualitymanagementkind) {
this.qualitymanagementkind = qualitymanagementkind;
}
public String getPidsystems() {
return pidsystems;
}
public void setPidsystems(final String pidsystems) {
this.pidsystems = pidsystems;
}
public String getCertificates() {
return certificates;
}
public void setCertificates(final String certificates) {
this.certificates = certificates;
}
public String getEoscType() {
return eoscType;
}
public void setEoscType(final String eoscType) {
this.eoscType = eoscType;
}
public Boolean getDedupMainService() {
return dedupMainService;
}
public void setDedupMainService(final Boolean dedupMainService) {
this.dedupMainService = dedupMainService;
}
}

View File

@ -0,0 +1,114 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Datasource info model", description = "provides information about the datasource and its aggregation status")
public class DatasourceInfo {
private long indexRecords;
private long fundedContent;
private long fulltexts;
private String lastIndexingDate;
private String firstHarvestDate;
private DatasourceDetails datasource;
private AggregationInfo lastCollection;
private AggregationInfo lastTransformation;
private List<AggregationInfo> aggregationHistory;
public DatasourceInfo() {
super();
}
public DatasourceInfo setIndexRecords(final long indexRecords) {
this.indexRecords = indexRecords;
return this;
}
public DatasourceInfo setFundedContent(final long fundedContent) {
this.fundedContent = fundedContent;
return this;
}
public DatasourceInfo setFulltexts(final long fulltexts) {
this.fulltexts = fulltexts;
return this;
}
public DatasourceInfo setLastIndexingDate(final String lastIndexingDate) {
this.lastIndexingDate = lastIndexingDate;
return this;
}
public DatasourceInfo setAggregationHistory(final List<AggregationInfo> aggregationHistory) {
this.aggregationHistory = aggregationHistory;
return this;
}
public DatasourceInfo setLastCollection(final AggregationInfo lastCollection) {
this.lastCollection = lastCollection;
return this;
}
public DatasourceInfo setLastTransformation(final AggregationInfo lastTransformation) {
this.lastTransformation = lastTransformation;
return this;
}
public long getIndexRecords() {
return indexRecords;
}
public long getFundedContent() {
return fundedContent;
}
public long getFulltexts() {
return fulltexts;
}
public String getLastIndexingDate() {
return lastIndexingDate;
}
public List<AggregationInfo> getAggregationHistory() {
return aggregationHistory;
}
public AggregationInfo getLastCollection() {
return lastCollection;
}
public AggregationInfo getLastTransformation() {
return lastTransformation;
}
public DatasourceDetails getDatasource() {
return datasource;
}
public DatasourceInfo setDatasource(final DatasourceDetails datasource) {
this.datasource = datasource;
return this;
}
public String getFirstHarvestDate() {
return firstHarvestDate;
}
public void setFirstHarvestDate(final String firstHarvestDate) {
this.firstHarvestDate = firstHarvestDate;
}
}

View File

@ -0,0 +1,27 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.common.collect.Lists;
@JsonAutoDetect
public class DatasourceResponse<T> extends Response {
private List<T> datasourceInfo = Lists.newArrayList();
public DatasourceResponse<T> addDatasourceInfo(final T datasourceInfo) {
getDatasourceInfo().add(datasourceInfo);
return this;
}
public List<T> getDatasourceInfo() {
return datasourceInfo;
}
public DatasourceResponse<T> setDatasourceInfo(final List<T> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
return this;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class DatasourceSearchResponse extends Response {
private List<DatasourceInfo> datasourceInfo;
public DatasourceSearchResponse(final List<DatasourceInfo> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceInfo> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(final List<DatasourceInfo> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

View File

@ -0,0 +1,45 @@
package eu.dnetlib.openaire.dsm.domain;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Datasource model", description = "provides information about the datasource")
public class DatasourceSnippet {
@NotBlank
private String id;
@NotBlank
private String officialname;
@NotBlank
private String englishname;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public String getEnglishname() {
return englishname;
}
public void setEnglishname(final String englishname) {
this.englishname = englishname;
}
}

View File

@ -0,0 +1,182 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.Date;
import java.util.Set;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Datasource model", description = "provides extended information about the datasource")
public class DatasourceSnippetExtended {
@NotBlank
private String id;
@NotBlank
private String officialname;
@NotBlank
private String englishname;
private String websiteurl;
@Email
private String registeredby;
private Date registrationdate;
private String eoscDatasourceType;
private String logoUrl;
private String description;
private Boolean consentTermsOfUse;
private Date consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate;
private Boolean fullTextDownload;
private Set<OrganizationDetails> organizations;
@Deprecated
private String typology;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialname() {
return officialname;
}
public void setOfficialname(final String officialname) {
this.officialname = officialname;
}
public String getEnglishname() {
return englishname;
}
public void setEnglishname(final String englishname) {
this.englishname = englishname;
}
public String getWebsiteurl() {
return websiteurl;
}
public void setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
}
public String getRegisteredby() {
return registeredby;
}
public void setRegisteredby(final String registeredby) {
this.registeredby = registeredby;
}
public Date getRegistrationdate() {
return registrationdate;
}
public void setRegistrationdate(final Date registrationdate) {
this.registrationdate = registrationdate;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getLogoUrl() {
return logoUrl;
}
public Boolean getConsentTermsOfUse() {
return consentTermsOfUse;
}
public DatasourceSnippetExtended setConsentTermsOfUse(final Boolean consentTermsOfUse) {
this.consentTermsOfUse = consentTermsOfUse;
return this;
}
public Date getConsentTermsOfUseDate() {
return consentTermsOfUseDate;
}
public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate;
return this;
}
public Boolean getFullTextDownload() {
return fullTextDownload;
}
public DatasourceSnippetExtended setFullTextDownload(final Boolean fullTextDownload) {
this.fullTextDownload = fullTextDownload;
return this;
}
public DatasourceSnippetExtended setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
return this;
}
public String getDescription() {
return description;
}
public DatasourceSnippetExtended setDescription(final String description) {
this.description = description;
return this;
}
public Set<OrganizationDetails> getOrganizations() {
return organizations;
}
public DatasourceSnippetExtended setOrganizations(final Set<OrganizationDetails> organizations) {
this.organizations = organizations;
return this;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public DatasourceSnippetExtended setTypology(final String typology) {
this.typology = typology;
return this;
}
public Date getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate;
}
public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this;
}
}

View File

@ -0,0 +1,24 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class DatasourceSnippetResponse extends Response {
private List<DatasourceSnippetExtended> datasourceInfo;
public DatasourceSnippetResponse(final List<DatasourceSnippetExtended> datasourceInfo) {
super();
this.datasourceInfo = datasourceInfo;
}
public List<DatasourceSnippetExtended> getDatasourceInfo() {
return datasourceInfo;
}
public void setDatasourceInfo(final List<DatasourceSnippetExtended> datasourceInfo) {
this.datasourceInfo = datasourceInfo;
}
}

View File

@ -0,0 +1,47 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Filter name", description = "List of the field names used to filter datasources")
public enum FilterName {
id,
managed,
collectedfrom, // exact match
officialname,
englishname,
websiteurl,
contactemail,
registeredby,
typology,
eoscDatasourceType,
platform, // like match
country; // exact match on related organization
public static FilterType type(final FilterName filterName) {
switch (filterName) {
case id:
case managed:
case collectedfrom:
return FilterType.exact;
case officialname:
case englishname:
case websiteurl:
case contactemail:
case registeredby:
case typology:
case platform:
return FilterType.search;
case eoscDatasourceType:
return FilterType.multiSearch;
case country:
return FilterType.searchOrgs;
default:
throw new IllegalStateException("unmapped filter type for: " + filterName);
}
}
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
public enum FilterType {
exact,
search,
searchOrgs,
multiSearch
}

View File

@ -0,0 +1,110 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import java.util.Queue;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
@JsonAutoDetect
public class Header {
private long total;
private int page;
private int size;
private long time;
private int statusCode;
private List<String> errors = Lists.newArrayList();
@JsonIgnore
private Queue<Throwable> exceptions = Lists.newLinkedList();
public static Header newInsance() {
return new Header();
}
public Header() {}
public long getTime() {
return time;
}
public Header setTime(final long time) {
this.time = time;
return this;
}
public int getStatusCode() {
return statusCode;
}
public Header setStatusCode(final int statusCode) {
this.statusCode = statusCode;
return this;
}
public long getTotal() {
return total;
}
public int getPage() {
return page;
}
public int getSize() {
return size;
}
public Header setPage(final int page) {
this.page = page;
return this;
}
public Header setSize(final int size) {
this.size = size;
return this;
}
public Header setTotal(final long total) {
this.total = total;
return this;
}
public Queue<Throwable> getExceptions() {
return exceptions;
}
public Header setExceptions(final Queue<Throwable> exceptions) {
this.exceptions = exceptions;
return this;
}
public List<String> getErrors() {
return getExceptions().stream()
.map(Throwable::getMessage)
.collect(Collectors.toList());
}
public Header setErrors(final List<String> errors) {
this.errors = errors;
return this;
}
public String toJson() {
try {
return new ObjectMapper().writeValueAsString(this);
} catch (final JsonProcessingException e) {
throw new RuntimeException(e);
}
}
}

View File

@ -0,0 +1,30 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class IdentitiesDetails {
private String pid;
private String issuertype;
public String getPid() {
return pid;
}
public String getIssuertype() {
return issuertype;
}
public IdentitiesDetails setPid(final String pid) {
this.pid = pid;
return this;
}
public IdentitiesDetails setIssuertype(final String issuertype) {
this.issuertype = issuertype;
return this;
}
}

View File

@ -0,0 +1,70 @@
package eu.dnetlib.openaire.dsm.domain;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Organization info model", description = "provides information about the organization")
public class OrganizationDetails extends OrganizationIgnoredProperties {
private String legalshortname;
@NotBlank
private String legalname;
private String websiteurl;
private String logourl;
@NotBlank
private String country;
public String getLegalshortname() {
return legalshortname;
}
public String getLegalname() {
return legalname;
}
public String getWebsiteurl() {
return websiteurl;
}
public String getLogourl() {
return logourl;
}
public String getCountry() {
return country;
}
public OrganizationDetails setLegalshortname(final String legalshortname) {
this.legalshortname = legalshortname;
return this;
}
public OrganizationDetails setLegalname(final String legalname) {
this.legalname = legalname;
return this;
}
public OrganizationDetails setWebsiteurl(final String websiteurl) {
this.websiteurl = websiteurl;
return this;
}
public OrganizationDetails setLogourl(final String logourl) {
this.logourl = logourl;
return this;
}
public OrganizationDetails setCountry(final String country) {
this.country = country;
return this;
}
}

View File

@ -0,0 +1,64 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.Date;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore;
public class OrganizationIgnoredProperties {
@JsonIgnore
protected String id;
@JsonIgnore
protected String collectedfrom;
@JsonIgnore
protected Date dateofcollection;
@JsonIgnore
protected String provenanceaction;
@JsonIgnore
protected Set<?> datasources;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getCollectedfrom() {
return collectedfrom;
}
public void setCollectedfrom(final String collectedfrom) {
this.collectedfrom = collectedfrom;
}
public Date getDateofcollection() {
return dateofcollection;
}
public void setDateofcollection(final Date dateofcollection) {
this.dateofcollection = dateofcollection;
}
public String getProvenanceaction() {
return provenanceaction;
}
public void setProvenanceaction(final String provenanceaction) {
this.provenanceaction = provenanceaction;
}
public Set<?> getDatasources() {
return datasources;
}
public void setDatasources(final Set<?> datasources) {
this.datasources = datasources;
}
}

View File

@ -0,0 +1,96 @@
package eu.dnetlib.openaire.dsm.domain;
public class RegisteredDatasourceInfo {
private String id;
private String officialName;
private String englishName;
private String organization;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
private String lastCollectionDate;
private long lastCollectionTotal;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialName() {
return officialName;
}
public void setOfficialName(final String officialName) {
this.officialName = officialName;
}
public String getEnglishName() {
return englishName;
}
public void setEnglishName(final String englishName) {
this.englishName = englishName;
}
public String getOrganization() {
return organization;
}
public void setOrganization(final String organization) {
this.organization = organization;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
return registeredBy;
}
public void setRegisteredBy(final String registeredBy) {
this.registeredBy = registeredBy;
}
public String getRegistrationDate() {
return registrationDate;
}
public void setRegistrationDate(final String registrationDate) {
this.registrationDate = registrationDate;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(final String compatibility) {
this.compatibility = compatibility;
}
public String getLastCollectionDate() {
return lastCollectionDate;
}
public void setLastCollectionDate(final String lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
}
public long getLastCollectionTotal() {
return lastCollectionTotal;
}
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

View File

@ -0,0 +1,20 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.HashMap;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Request filter", description = "field name and value pairs")
public class RequestFilter extends HashMap<FilterName, Object> {
/**
*
*/
private static final long serialVersionUID = 5501969842482508379L;
public RequestFilter() {}
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public enum RequestSort {
id, officialname, dateofvalidation, registrationdate
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public enum RequestSortOrder {
ASCENDING, DESCENDING
}

View File

@ -0,0 +1,26 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect
@Schema(name = "Api response model", description = "Api response model, provides a response header")
public class Response {
private Header header;
public Response() {
this.header = new Header();
}
public Header getHeader() {
return header;
}
public Response setHeader(final Header header) {
this.header = header;
return this;
}
}

View File

@ -0,0 +1,129 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.LinkedHashMap;
import java.util.Map;
public class SimpleDatasourceInfo {
private String id;
private String officialName;
private String englishName;
private Map<String, String> organizations = new LinkedHashMap<>();
@Deprecated
private String typology;
private String eoscType;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
private String firstCollectionDate;
private String lastCollectionDate;
private long lastCollectionTotal;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialName() {
return officialName;
}
public void setOfficialName(final String officialName) {
this.officialName = officialName;
}
public String getEnglishName() {
return englishName;
}
public void setEnglishName(final String englishName) {
this.englishName = englishName;
}
public Map<String, String> getOrganizations() {
return organizations;
}
public void setOrganizations(final Map<String, String> organizations) {
this.organizations = organizations;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public void setTypology(final String typology) {
this.typology = typology;
}
public String getEoscType() {
return eoscType;
}
public void setEoscType(final String eoscType) {
this.eoscType = eoscType;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
return registeredBy;
}
public void setRegisteredBy(final String registeredBy) {
this.registeredBy = registeredBy;
}
public String getRegistrationDate() {
return registrationDate;
}
public void setRegistrationDate(final String registrationDate) {
this.registrationDate = registrationDate;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(final String compatibility) {
this.compatibility = compatibility;
}
public String getFirstCollectionDate() {
return firstCollectionDate;
}
public void setFirstCollectionDate(final String firstCollectionDate) {
this.firstCollectionDate = firstCollectionDate;
}
public String getLastCollectionDate() {
return lastCollectionDate;
}
public void setLastCollectionDate(final String lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
}
public long getLastCollectionTotal() {
return lastCollectionTotal;
}
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

View File

@ -0,0 +1,20 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@JsonAutoDetect
public class SimpleResponse<T> extends Response {
private List<T> response;
public List<T> getResponse() {
return response;
}
public SimpleResponse<T> setResponse(final List<T> response) {
this.response = response;
return this;
}
}

View File

@ -0,0 +1,11 @@
package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
public class TransformationInfo extends AggregationInfo {
}

View File

@ -0,0 +1,165 @@
package eu.dnetlib.openaire.dsm.utils;
import java.util.Arrays;
import java.util.List;
import java.util.Map.Entry;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.data.jpa.domain.Specification;
import eu.dnetlib.data.openaire.dsm.model.Datasource;
import eu.dnetlib.data.openaire.dsm.model.DatasourceApi;
import eu.dnetlib.openaire.dsm.domain.FilterName;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
public class DatasourceSpecs {
private static final Log log = LogFactory.getLog(DatasourceSpecs.class);
public static final String WILDCARD = "%";
public static Specification<Datasource> dsRegisteredbyNotNullSpec() {
return (ds, query, cb) -> cb.and(cb.isNull(ds.get(FilterName.registeredby.name())).not(), cb.isNull(ds.get("registrationdate")).not());
}
public static Specification<Datasource> dsSpec(final RequestSort requestSortBy, final RequestSortOrder order, final RequestFilter requestFilter) {
log.debug(String.format("RequestFilter:'%s', RequestSort:'%s', RequestSortOrder:'%s'", requestFilter, requestSortBy, order));
return (ds, query, cb) -> {
final Predicate p = cb.conjunction();
final List<Expression<Boolean>> expressions = p.getExpressions();
expressions.add(cb.equal(ds.get("eoscType"), "Data Source"));
expressions.add(cb.equal(ds.get("dedupMainService"), true));
if (requestFilter != null) {
requestFilter.entrySet()
.stream()
.forEach(e -> {
switch (FilterName.type(e.getKey())) {
case exact:
expressions.add(exactSearch(ds, cb, e));
break;
case search:
expressions.add(likeSearch(ds, cb, e));
break;
case multiSearch:
expressions.add(likeSearchList(ds, cb, e));
break;
case searchOrgs:
// search by case insensitive organization's country
expressions.add(cb.equal(cb.lower(ds.join("organizations").get(FilterName.country.name())), getValue(e)));
break;
}
});
}
if (requestSortBy != null) {
if (order != null) {
final Path<Object> orderField = ds.get(requestSortBy.name());
switch (order) {
case ASCENDING:
query.orderBy(cb.asc(orderField));
break;
case DESCENDING:
query.orderBy(cb.desc(orderField));
break;
}
}
}
query.distinct(true);
return p;
};
}
public static Specification<DatasourceApi> apiSpec(final RequestFilter requestFilter) {
log.debug(String.format("RequestFilter:'%s'", requestFilter));
return (api, query, cb) -> {
final Predicate p = cb.conjunction();
if (requestFilter != null) {
final List<Expression<Boolean>> expressions = p.getExpressions();
requestFilter.entrySet()
.stream()
.forEach(e -> {
switch (FilterName.type(e.getKey())) {
case exact:
expressions.add(exactSearch(api, cb, e));
break;
case search:
expressions.add(likeSearch(api, cb, e));
break;
case multiSearch:
expressions.add(likeSearchList(api, cb, e));
break;
case searchOrgs:
throw new RuntimeException("not implemented");
}
});
}
query.distinct(true);
return p;
};
}
// HELPERS
// substring, case insensitive, like based search
private static Predicate likeSearch(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
return cb.like(cb.lower(r.get(e.getKey().name())), WILDCARD + getValue(e) + WILDCARD);
}
// substring, case insensitive, like based search
private static Predicate likeSearchList(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
final String key = e.getKey().name();
final Predicate[] arr =
Arrays.stream(e.getValue().toString().split(","))
.map(String::trim)
.map(String::toLowerCase)
.filter(StringUtils::isNotBlank)
.map(s -> cb.like(cb.lower(r.get(key)), WILDCARD + s + WILDCARD))
.toArray(size -> new Predicate[size]);
return cb.or(arr);
}
// search by ID, managed. exact match
private static Predicate exactSearch(final Root<?> r, final CriteriaBuilder cb, final Entry<FilterName, Object> e) {
return cb.equal(r.get(e.getKey().name()), getValue(e));
}
private static Object getValue(final Entry<FilterName, Object> e) {
if (e.getValue() instanceof String) {
final String s = (String) e.getValue();
if (!e.getKey().equals(FilterName.country)) {
final Boolean b = BooleanUtils.toBooleanObject(s);
if (b != null) { return b; }
}
return e.getKey().equals(FilterName.id) ? s : StringUtils.lowerCase(s);
}
if (e.getValue() instanceof Boolean) { return e.getValue(); }
return e.getValue();
}
}

View File

@ -0,0 +1,190 @@
package eu.dnetlib.openaire.dsm.utils;
import java.sql.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.BeanWrapper;
import org.springframework.beans.BeanWrapperImpl;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.data.openaire.dsm.model.Api;
import eu.dnetlib.data.openaire.dsm.model.Datasource;
import eu.dnetlib.data.openaire.dsm.model.Organization;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended;
import eu.dnetlib.openaire.dsm.domain.OrganizationDetails;
public class DsmMappingUtils {
public static final String DATE_FORMAT = "yyyy-MM-dd";
public static final String ID_SEPARATOR = "::";
public static final String ID_PREFIX = "api_________" + ID_SEPARATOR;
public static String createId(final ApiDetails api) {
return ID_PREFIX + api.getDatasource() + ID_SEPARATOR + RandomStringUtils.randomAlphanumeric(8);
}
public static DatasourceDetails asDetails(final Datasource d) {
final DatasourceDetails details = _convert(d, DatasourceDetails.class);
return details.setOpenaireId(asOpenaireId(details.getId()));
}
public static DatasourceSnippetExtended asSnippetExtended(final Datasource d) {
final DatasourceSnippetExtended ds = new DatasourceSnippetExtended();
ds.setId(d.getId());
ds.setOfficialname(d.getOfficialname());
ds.setEnglishname(d.getEnglishname());
ds.setRegisteredby(d.getRegisteredby());
ds.setWebsiteurl(d.getWebsiteurl());
ds.setEoscDatasourceType(d.getEoscDatasourceType());
ds.setRegistrationdate(d.getRegistrationdate());
ds.setLogoUrl(d.getLogourl());
ds.setDescription(d.getDescription());
ds.setConsentTermsOfUse(d.getConsentTermsOfUse());
ds.setConsentTermsOfUseDate(d.getConsentTermsOfUseDate());
ds.setLastConsentTermsOfUseDate(d.getLastConsentTermsOfUseDate());
ds.setFullTextDownload(d.getFullTextDownload());
if (d.getOrganizations() != null) {
ds.setOrganizations(d.getOrganizations().stream().map(DsmMappingUtils::asOrganizationDetail).collect(Collectors.toSet()));
}
ds.setTypology(d.getTypology());
return ds;
}
private static OrganizationDetails asOrganizationDetail(final Organization o) {
return new OrganizationDetails()
.setCountry(o.getCountry())
.setLegalname(o.getLegalname())
.setLegalshortname(o.getLegalshortname())
.setWebsiteurl(o.getWebsiteurl())
.setLogourl(o.getLogourl());
}
public static ApiDetails asDetails(final Api d) {
return _convert(d, ApiDetails.class);
}
public static Api asDbEntry(final ApiDetails d) {
final Api apiDbEntry = new Api();
apiDbEntry.setId(d.getId());
apiDbEntry.setBaseurl(d.getBaseurl());
apiDbEntry.setProtocol(d.getProtocol());
apiDbEntry.setDatasource(d.getDatasource());
apiDbEntry.setContentdescription(d.getContentdescription());
apiDbEntry.setCompatibility(d.getCompatibility());
apiDbEntry.setCompatibilityOverride(d.getCompatibilityOverride());
apiDbEntry.setRemovable(d.getRemovable());
apiDbEntry.setMetadataIdentifierPath(d.getMetadataIdentifierPath());
return apiDbEntry;
}
public static Datasource asDbEntry(final DatasourceDetails d) {
final Datasource dbe = _convert(d, Datasource.class);
if (dbe.getOrganizations() != null) {
dbe.getOrganizations().forEach(o -> {
final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId();
o.setId(prefix + ID_SEPARATOR + o.getLegalname());
if (o.getDateofcollection() == null) {
o.setDateofcollection(new Date(System.currentTimeMillis()));
}
o.setCollectedfrom(dbe.getCollectedfrom());
});
}
_fix_typology(dbe, d.getTypology(), d.getEoscDatasourceType());
return dbe;
}
@Deprecated
private static void _fix_typology(final Datasource dbe, final String oldTypology, final String eoscTypology) {
if (StringUtils.isNotBlank(oldTypology) && StringUtils.isBlank(eoscTypology)) {
// THE ORDER IS IMPORTANT: DO NOT CHANGE IT
if (oldTypology.startsWith("crissystem")) {
dbe.setEoscDatasourceType("CRIS system");
} else if (oldTypology.startsWith("entityregistry")) {
dbe.setEoscDatasourceType("Registry");
} else if (oldTypology.startsWith("pubscatalogue") || oldTypology.equals("websource")) {
dbe.setEoscDatasourceType("Catalogue");
} else if (oldTypology.contains("journal")) {
dbe.setEoscDatasourceType("Journal archive");
} else if (oldTypology.startsWith("aggregator")) {
dbe.setEoscDatasourceType("Aggregator");
} else if (oldTypology.contains("repository")) {
dbe.setEoscDatasourceType("Repository");
} else {
dbe.setEoscDatasourceType("Aggregator");
}
} else if (StringUtils.isBlank(oldTypology) && StringUtils.isNotBlank(eoscTypology)) {
if (dbe.getEoscDatasourceType().equals("CRIS system")) {
dbe.setTypology("crissystem");
} else if (dbe.getEoscDatasourceType().equals("Registry")) {
dbe.setTypology("entityregistry");
} else if (dbe.getEoscDatasourceType().equals("Catalogue")) {
dbe.setTypology("pubscatalogue::unknown");
} else if (dbe.getEoscDatasourceType().equals("Journal archive")) {
dbe.setTypology("pubsrepository::journal");
} else if (dbe.getEoscDatasourceType().equals("Aggregator")) {
dbe.setTypology("aggregator");
} else if (dbe.getEoscDatasourceType().equals("Repository")) {
dbe.setTypology("pubsrepository::unknown");
} else {
dbe.setTypology("aggregator");
}
}
}
public static Datasource asDbEntry(final DatasourceDetailsUpdate d) {
final Datasource dbe = _convert(d, Datasource.class);
_fix_typology(dbe, d.getTypology(), d.getEoscDatasourceType());
return dbe;
}
// HELPERS
private static <T> T _convert(final Object o, final Class<T> clazz) {
final ObjectMapper mapper = new ObjectMapper();
return mapper.convertValue(o, clazz);
}
private static String asOpenaireId(final String id) {
final String prefix = StringUtils.substringBefore(id, ID_SEPARATOR);
final String md5 = StringUtils.substringAfter(id, ID_SEPARATOR);
return prefix + ID_SEPARATOR + DigestUtils.md5Hex(md5);
}
public static void copyNonNullProperties(final Object src, final Object target) {
BeanUtils.copyProperties(src, target, getNullPropertyNames(src));
}
public static String[] getNullPropertyNames(final Object source) {
final BeanWrapper src = new BeanWrapperImpl(source);
final java.beans.PropertyDescriptor[] pds = src.getPropertyDescriptors();
final Set<String> emptyNames = new HashSet<>();
for (final java.beans.PropertyDescriptor pd : pds) {
final Object srcValue = src.getPropertyValue(pd.getName());
if (srcValue == null) {
emptyNames.add(pd.getName());
}
}
final String[] result = new String[emptyNames.size()];
return emptyNames.toArray(result);
}
}

View File

@ -0,0 +1,61 @@
package eu.dnetlib.openaire.dsm.utils;
import java.util.List;
import java.util.Queue;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.DatasourceSearchResponse;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetExtended;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.Header;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
public class ResponseUtils {
public static ApiDetailsResponse apiResponse(final List<ApiDetails> api, final long total) {
final ApiDetailsResponse rsp = new ApiDetailsResponse().setApi(api);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceSnippetResponse snippetResponse(final List<DatasourceSnippetExtended> snippets, final long total) {
final DatasourceSnippetResponse rsp = new DatasourceSnippetResponse(snippets);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceDetailResponse detailsResponse(final List<DatasourceDetails> details, final long total) {
final DatasourceDetailResponse rsp = new DatasourceDetailResponse(details);
rsp.setHeader(header(total));
return rsp;
}
public static DatasourceSearchResponse searchResponse(final List<DatasourceInfo> infos, final long total) {
final DatasourceSearchResponse rsp = new DatasourceSearchResponse(infos);
rsp.setHeader(header(total));
return rsp;
}
public static Header header(final Queue<Throwable> errors, final long total) {
return Header.newInsance()
.setExceptions(errors)
.setTotal(total);
}
public static Header header(final long total) {
return header(Lists.newLinkedList(), total);
}
public static <T> SimpleResponse<T> simpleResponse(final List<T> list) {
final SimpleResponse<T> rsp = new SimpleResponse<T>().setResponse(list);;
rsp.setHeader(header(Lists.newLinkedList(), list.size()));
return rsp;
}
}

View File

@ -0,0 +1,30 @@
package eu.dnetlib.openaire.dsm.utils;
import org.springframework.stereotype.Component;
import eu.dnetlib.data.is.vocabulary.model.Vocabulary;
@Component
public class VocabularyClient {
public void dropCache() {
// TODO Auto-generated method stub
}
public Vocabulary getCountries() {
// TODO Auto-generated method stub
return null;
}
public Vocabulary getDatasourceTypologies() {
// TODO Auto-generated method stub
return null;
}
public boolean isValidTerm(final String string, final String type) {
// TODO Auto-generated method stub
return false;
}
}

View File

@ -0,0 +1,33 @@
package eu.dnetlib.openaire.dsm.utils;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import eu.dnetlib.is.errors.DsmException;
import eu.dnetlib.openaire.dsm.domain.AggregationInfo;
/**
* Created by claudio on 20/10/2016.
*/
@Component
@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
public class WfLoggerClient {
private static final Log log = LogFactory.getLog(WfLoggerClient.class);
private final static String LOADTIME = "loadtime";
public List<AggregationInfo> getAggregationHistory(final String dsId) throws DsmException {
// TODO Auto-generated method stub
return null;
}
public void dropCache() {
// TODO Auto-generated method stub
}
}

View File

@ -14,11 +14,20 @@ management.endpoints.web.base-path = /
management.endpoints.web.path-mapping.prometheus = metrics
management.endpoints.web.path-mapping.health = health
spring.datasource.url=jdbc:postgresql://localhost:5432/new_is
spring.datasource.username=
spring.datasource.password=
# Main Database for IS
spring.datasource.jdbc-url=jdbc:postgresql://localhost:5432/new_is
spring.datasource.username=dnet
spring.datasource.password=dnetPwd
spring.datasource.driver-class-name=org.postgresql.Driver
# Database for Main Entities
openaire.datasource.jdbc-url=jdbc:postgresql://localhost:5432/dnet_openaireplus
openaire.datasource.username=dnet
openaire.datasource.password=dnetPwd
openaire.datasource.driver-class-name=org.postgresql.Driver
# Hibernate ddl auto (create, create-drop, validate, update)
spring.jpa.database=default
spring.jpa.hibernate.ddl-auto = validate
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW
@ -27,3 +36,7 @@ spring.jpa.open-in-view=true
spring.jpa.properties.hibernate.show_sql=false
spring.jpa.properties.hibernate.use_sql_comments=false
spring.jpa.properties.hibernate.format_sql=false
openaire.api.enable.dsm = true

View File

@ -0,0 +1,7 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

View File

@ -0,0 +1,8 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
where d._typology_to_remove_ like ?
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

View File

@ -0,0 +1,29 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

View File

@ -0,0 +1,31 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s._typology_to_remove_ like ?
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

View File

@ -0,0 +1,11 @@
SELECT
funder,
jurisdiction,
fundingpathid,
acronym,
title,
code,
startdate,
enddate
FROM projects_api
WHERE fundingpathid like '$fundingprefix$%'

View File

@ -0,0 +1,25 @@
select
d.id as id,
d.officialname as "officialName",
d.englishname as "englishName",
o.legalname as organization,
d.eosc_datasource_type as eosc_datasource_type,
d.registeredby as "registeredBy",
d.registrationdate::text as "registrationDate",
a.compatibility as compatibility,
a.last_collection_date as "lastCollectionDate",
a.last_collection_total as "lastCollectionTotal"
from
dsm_services d
left outer join dsm_api a on (d.id = a.service)
left outer join dsm_service_organization dso on (d.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
where
d.registrationdate is not null
and d.registrationdate < a.last_collection_date
and d.registeredby is not null
and d.managed = true
and a.last_collection_total > 0
and a.active = true
order by d.registrationdate desc
limit ?;

View File

@ -0,0 +1,11 @@
select count(DISTINCT d.id) as count
from
dsm_services d
left outer join dsm_api a on (d.id = a.service)
where
d.registrationdate >= cast(? as date)
and d.registrationdate < a.last_collection_date
and d.registeredby is not null
and d.managed = true
and a.last_collection_total > 0
and a.active = true;

View File

@ -0,0 +1,12 @@
select count(DISTINCT d.id) as count
from
dsm_services d
left outer join dsm_api a on (d.id = a.service)
where
d.registrationdate >= cast(? as date)
and d._typology_to_remove_ like ?
and d.registrationdate < a.last_collection_date
and d.registeredby is not null
and d.managed = true
and a.last_collection_total > 0
and a.active = true;

View File

@ -0,0 +1,38 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s.registrationdate is not null
and s.registeredby is not null
and s.managed = true
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING
s.registrationdate < max(a.last_collection_date)
and sum(a.last_collection_total) > 0
ORDER BY s.registrationdate desc
LIMIT ?

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.common;
package eu.dnetlib.data.is.common;
import java.util.Optional;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Entity;
import javax.persistence.Table;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Entity;
import javax.persistence.Table;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Entity;
import javax.persistence.Table;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Entity;
import javax.persistence.Table;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Column;
import javax.persistence.Entity;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import javax.persistence.Column;
import javax.persistence.MappedSuperclass;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import java.io.Serializable;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.context.model;
package eu.dnetlib.data.is.context.model;
import java.io.Serializable;
import java.util.Objects;

View File

@ -1,8 +1,8 @@
package eu.dnetlib.is.context.model.repository;
package eu.dnetlib.data.is.context.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.is.context.model.Category;
import eu.dnetlib.data.is.context.model.Category;
public interface CategoryRepository extends JpaRepository<Category, String> {

View File

@ -1,8 +1,8 @@
package eu.dnetlib.is.context.model.repository;
package eu.dnetlib.data.is.context.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.is.context.model.ConceptLevel0;
import eu.dnetlib.data.is.context.model.ConceptLevel0;
public interface ConceptLevel0Repository extends JpaRepository<ConceptLevel0, String> {

View File

@ -1,8 +1,8 @@
package eu.dnetlib.is.context.model.repository;
package eu.dnetlib.data.is.context.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.is.context.model.ConceptLevel1;
import eu.dnetlib.data.is.context.model.ConceptLevel1;
public interface ConceptLevel1Repository extends JpaRepository<ConceptLevel1, String> {

View File

@ -1,8 +1,8 @@
package eu.dnetlib.is.context.model.repository;
package eu.dnetlib.data.is.context.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.is.context.model.ConceptLevel2;
import eu.dnetlib.data.is.context.model.ConceptLevel2;
public interface ConceptLevel2Repository extends JpaRepository<ConceptLevel2, String> {

View File

@ -1,8 +1,8 @@
package eu.dnetlib.is.context.model.repository;
package eu.dnetlib.data.is.context.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.is.context.model.Context;
import eu.dnetlib.data.is.context.model.Context;
public interface ContextRepository extends JpaRepository<Context, String> {

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.resource.model;
package eu.dnetlib.data.is.resource.model;
import java.io.Serializable;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.resource.model;
package eu.dnetlib.data.is.resource.model;
import java.io.Serializable;
import java.util.Date;

View File

@ -0,0 +1,8 @@
package eu.dnetlib.data.is.resource.repository;
import eu.dnetlib.data.is.common.ReadOnlyRepository;
import eu.dnetlib.data.is.resource.model.ResourceType;
public interface ResourceTypeRepository extends ReadOnlyRepository<ResourceType, String> {
}

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.resource.repository;
package eu.dnetlib.data.is.resource.repository;
import java.util.List;
import java.util.Optional;
@ -7,7 +7,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import eu.dnetlib.is.resource.model.SimpleResource;
import eu.dnetlib.data.is.resource.model.SimpleResource;
public interface SimpleResourceRepository extends JpaRepository<SimpleResource, String> {

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.vocabulary.model;
package eu.dnetlib.data.is.vocabulary.model;
import java.io.Serializable;
import java.util.Objects;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.vocabulary.model;
package eu.dnetlib.data.is.vocabulary.model;
import java.io.Serializable;

View File

@ -1,4 +1,4 @@
package eu.dnetlib.is.vocabulary.model;
package eu.dnetlib.data.is.vocabulary.model;
import java.io.Serializable;

Some files were not shown because too many files have changed in this diff Show More