dnet-applications/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java

537 lines
20 KiB
Java

package eu.dnetlib.dsm;
import static eu.dnetlib.dsm.utils.DatasourceSpecs.apiSpec;
import static eu.dnetlib.dsm.utils.DatasourceSpecs.dsRegisteredbyNotNullSpec;
import static eu.dnetlib.dsm.utils.DatasourceSpecs.dsSpec;
import static eu.dnetlib.dsm.utils.DsmMappingUtils.asDbEntry;
import static eu.dnetlib.dsm.utils.DsmMappingUtils.createId;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.EntityNotFoundException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.google.common.collect.Lists;
import eu.dnetlib.dsm.domain.ApiDetails;
import eu.dnetlib.dsm.domain.Country;
import eu.dnetlib.dsm.domain.DatasourceDetailResponse;
import eu.dnetlib.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.dsm.domain.RequestFilter;
import eu.dnetlib.dsm.domain.RequestSort;
import eu.dnetlib.dsm.domain.RequestSortOrder;
import eu.dnetlib.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.dsm.domain.SimpleResponse;
import eu.dnetlib.dsm.model.Api;
import eu.dnetlib.dsm.model.ApiParam;
import eu.dnetlib.dsm.model.BrowseTerm;
import eu.dnetlib.dsm.model.Datasource;
import eu.dnetlib.dsm.model.readonly.ApiWithAdditionalInfo;
import eu.dnetlib.dsm.model.readonly.SimpleDsWithApis;
import eu.dnetlib.dsm.repository.ApiRepository;
import eu.dnetlib.dsm.repository.ApiWithAdditionalInfoRepository;
import eu.dnetlib.dsm.repository.DatasourceRepository;
import eu.dnetlib.dsm.repository.SimpleDsWithApisRepository;
import eu.dnetlib.dsm.utils.DsmBrowsableFields;
import eu.dnetlib.dsm.utils.DsmMappingUtils;
import eu.dnetlib.dsm.utils.ResponseUtils;
import eu.dnetlib.dsm.utils.VocabularyClient;
import eu.dnetlib.errors.DsmException;
import eu.dnetlib.errors.DsmForbiddenException;
import eu.dnetlib.errors.DsmNotFoundException;
import eu.dnetlib.is.model.vocabulary.Vocabulary;
@Service
@ConditionalOnProperty(value = "openaire.api.enable.dsm", havingValue = "true")
public class DsmService {
private static final Log log = LogFactory.getLog(DsmService.class);
public static final String OAI = "oai";
public static final String SET = "set";
@Autowired
private DatasourceRepository dsRepository;
@Autowired
private ApiRepository apiRepository;
@Autowired
private ApiWithAdditionalInfoRepository apiWithAdditionalInfoRepository;
@Autowired
private SimpleDsWithApisRepository simpleDsWithApisRepository;
@Autowired
private VocabularyClient vocabularyClient;
@Autowired
@Qualifier("openaireJdbcTemplate")
private JdbcTemplate jdbcTemplate;
public List<Country> listCountries() throws DsmException {
final List<Country> countries = Lists.newArrayList();
// TODO Usare solo vocabolari???
final Vocabulary v = vocabularyClient.getCountries();
countries.addAll(browseTerm(DsmBrowsableFields.country)
.stream()
.filter(Objects::nonNull)
.map(t -> new Country(t.getTerm(), t.getName()))
.collect(Collectors.toList()));
return countries;
}
@Transactional
public DatasourceSnippetResponse searchSnippet(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size) throws DsmException {
final Page<Datasource> dsPage = search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.snippetResponse(dsPage.map(DsmMappingUtils::asSnippetExtended).getContent(), dsPage.getTotalElements());
}
public DatasourceDetailResponse searchDetails(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size) throws DsmException {
final Page<Datasource> dsPage = search(requestSortBy, order, requestFilter, page, size);
return ResponseUtils.detailsResponse(dsPage.map(DsmMappingUtils::asDetails).getContent(), dsPage.getTotalElements());
}
private Page<Datasource> search(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<Datasource> spec = dsSpec(requestSortBy, order, requestFilter);
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
public Page<Datasource> searchRegistered(final RequestSort requestSortBy,
final RequestSortOrder order,
final RequestFilter requestFilter,
final int page,
final int size)
throws DsmException {
final Specification<Datasource> spec = dsSpec(requestSortBy, order, requestFilter).and(dsRegisteredbyNotNullSpec());
return dsRepository.findAll(spec, PageRequest.of(page, size));
}
public Datasource getDs(final String dsId) throws DsmException {
return dsRepository.findById(dsId).orElseThrow(() -> new DsmException("Datasource not found. ID: " + dsId));
}
public Datasource getDsByNsPrefix(final String prefix) throws DsmException {
return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix));
}
public void setManaged(final String id, final boolean managed) {
log.info(String.format("setting managed = '%s' for ds '%s'", managed, id));
dsRepository.setManaged(id, managed);
apiRepository.setRemovable(id, true);
}
public boolean isManaged(final String id) {
return dsRepository.isManaged(id);
}
public void updateCompliance(final String dsId, final String apiId, final String compliance, final boolean override) {
log.info(String.format("setting compatibility = '%s' for ds '%s'", compliance, apiId));
apiRepository.updateCompatibility(apiId, compliance);
}
public List<Api> getApis(final String dsId) {
return apiRepository.findByDatasource(dsId);
}
public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException {
final Api api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
try {
if (!api.getRemovable()) { throw new DsmForbiddenException("api is not removable"); }
apiRepository.deleteById(apiId);
log.info(String.format("deleted api '%s'", apiId));
} catch (final EntityNotFoundException e) {
throw new DsmNotFoundException("api not found");
}
}
public void addApi(final Api api) {
apiRepository.save(api);
}
public boolean existDs(final String dsId) throws DsmException {
return dsRepository.existsById(dsId);
}
public void saveDs(final Datasource d) {
log.info(String.format("saving datasource '%s'", d.getId()));
final Datasource datasource = dsRepository.save(d);
log.info(String.format("saved datasource '%s'", datasource.getId()));
ensureRegistrationDate(d.getId());
}
public void deleteDs(final String dsId) {
dsRepository.deleteById(dsId);
log.info(String.format("deleted datasource '%s'", dsId));
}
public void updateName(final String dsId, final String officialname, final String englishname) {
// TODO what if one of the two names is null or empty?
dsRepository.setDatasourcename(dsId, officialname, englishname);
}
public void updateLogoUrl(final String dsId, final String logourl) throws DsmException {
dsRepository.setLogoUrl(dsId, logourl);
}
public void updateCoordinates(final String dsId, final Double latitude, final Double longitude) {
dsRepository.setCoordinates(dsId, latitude, longitude);
}
public void updateApiBaseUrl(final String apiId, final String baseurl) {
apiRepository.setBaseurl(apiId, baseurl);
}
@Transactional
public boolean upsertApiOaiSet(final String apiId, final String oaiSet) throws DsmException, DsmNotFoundException {
final Api api = apiRepository.findById(apiId).orElseThrow(() -> new DsmNotFoundException("Api not found. ID: " + apiId));
if (OAI.equalsIgnoreCase(api.getProtocol())) {
final Set<ApiParam> apiParams = api.getApiParams();
if (!apiParams.stream().anyMatch(ap -> SET.equals(ap.getParam()))) {
apiRepository.addApiParam(apiId, SET, oaiSet);
log.info(String.format("added api '%s' oai set with '%s'", apiId, oaiSet));
return true;
} else {
apiRepository.updateOaiSet(apiId, oaiSet);
log.info(String.format("updated api '%s' oai set with '%s'", apiId, oaiSet));
return false;
}
} else {
throw new DsmException(String.format("won't add OAI set to a non OAI interface: '%s' has protocol '%s'", apiId, api.getProtocol()));
}
}
public List<String> findApiBaseURLs(final RequestFilter requestFilter, final int page, final int size) throws DsmException {
final PageRequest pageable = PageRequest.of(page, size);
final Specification<ApiWithAdditionalInfo> spec = apiSpec(requestFilter);
final Set<String> set = apiWithAdditionalInfoRepository.findAll(spec, pageable)
.getContent()
.stream()
.map(ApiWithAdditionalInfo::getBaseurl)
.filter(StringUtils::isNotBlank)
.collect(Collectors.toCollection(HashSet::new));
return Lists.newArrayList(set);
}
public void updateTimezone(final String dsId, final String timezone) {
dsRepository.setTimezone(dsId, timezone);
}
public void updateEoscDatasourceType(final String dsId, final String type) throws DsmException {
if (vocabularyClient.isValidTerm("eosc_datasource_types", type)) {
throw new DsmException(String.format("invalid datasource type '%s', provide one according to vocabulary eosc_datasource_types"));
}
dsRepository.setEoscDatasourceType(dsId, type);
}
public void updateRegisteringUser(final String dsId, final String registeredBy) throws DsmException {
ensureRegistrationDate(dsId);
dsRepository.setRegisteringUser(dsId, registeredBy);
}
public void updatePlatform(final String dsId, final String platform) throws DsmException {
dsRepository.setPlatform(dsId, platform);
}
public SimpleResponse<RegisteredDatasourceInfo> searchRecentRegistered(final int size) throws DsmException {
try {
final List<RegisteredDatasourceInfo> list =
querySql("recent_registered_datasources.sql.st", BeanPropertyRowMapper.newInstance(RegisteredDatasourceInfo.class), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public Long countRegisteredAfter(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate, typeFilter + "%");
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public SimpleResponse<SimpleDatasourceInfo> searchRecentRegisteredV2(final int size) throws DsmException {
try {
final List<SimpleDatasourceInfo> list = querySql("recent_registered_datasources_v2.sql.st", rowMapperForSimpleDatasourceInfo(), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw new DsmException("error searching recent datasources", e);
}
}
public Long countFirstCollect(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
return querySql("count_first_collected_datasources_fromDate_typology.st.sql", Long.class, typeFilter + "%", fromDate);
} else {
return querySql("count_first_collected_datasources_fromDate.st.sql", Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw new DsmException("error searching datasources using the first collection date", e);
}
}
public List<SimpleDatasourceInfo> getFirstCollectedAfter(final String fromDate, final String typeFilter) throws DsmException {
try {
if (StringUtils.isNotBlank(typeFilter)) {
return querySql("first_collected_datasources_fromDate_typology.st.sql", rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate);
} else {
return querySql("first_collected_datasources_fromDate.st.sql", rowMapperForSimpleDatasourceInfo(), fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw new DsmException("error searching datasources using the first collection date", e);
}
}
private <T> List<T> querySql(final String sqlFile, final RowMapper<T> rowMapper, final Object... params) throws IOException {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/dsm/" + sqlFile), StandardCharsets.UTF_8);
return jdbcTemplate.query(sql, rowMapper, params);
}
private <T> T querySql(final String sqlFile, final Class<T> clazz, final Object... params) throws IOException {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/dsm/" + sqlFile), StandardCharsets.UTF_8);
return jdbcTemplate.queryForObject(sql, clazz, params);
}
// HELPER
private void ensureRegistrationDate(final String dsId) {
if (!dsRepository.hasRegistrationdate(dsId)) {
log.info("setting registration date for datasource: " + dsId);
dsRepository.setRegistrationDate(dsId, LocalDate.now());
}
}
private RowMapper<SimpleDatasourceInfo> rowMapperForSimpleDatasourceInfo() {
return (rs, rowNum) -> {
final SimpleDatasourceInfo info = new SimpleDatasourceInfo();
info.setId(rs.getString("id"));
info.setOfficialName(rs.getString("officialName"));
info.setEnglishName(rs.getString("englishName"));
info.setTypology(rs.getString("typology"));
info.setEoscType(rs.getString("eoscType"));
info.setEoscDatasourceType(rs.getString("eoscDatasourceType"));
info.setRegisteredBy(rs.getString("registeredBy"));
info.setRegistrationDate(rs.getString("registrationDate"));
info.setFirstCollectionDate(rs.getString("firstCollectionDate"));
info.setLastCollectionDate(rs.getString("lastCollectionDate"));
info.setLastCollectionTotal(rs.getLong("lastCollectionTotal"));
final Set<String> compatibilities = new HashSet<>();
for (final String s : (String[]) rs.getArray("compatibilities").getArray()) {
compatibilities.add(s);
}
// The order of the condition is important
if (compatibilities.contains("openaire-cris_1.1")) {
info.setCompatibility("openaire-cris_1.1");
} else if (compatibilities.contains("openaire4.0")) {
info.setCompatibility("openaire4.0");
} else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) {
info.setCompatibility("driver-openaire2.0");
} else if (compatibilities.contains("driver")) {
info.setCompatibility("driver");
} else if (compatibilities.contains("openaire2.0")) {
info.setCompatibility("openaire2.0");
} else if (compatibilities.contains("openaire3.0")) {
info.setCompatibility("openaire3.0");
} else if (compatibilities.contains("openaire2.0_data")) {
info.setCompatibility("openaire2.0_data");
} else if (compatibilities.contains("native")) {
info.setCompatibility("native");
} else if (compatibilities.contains("hostedBy")) {
info.setCompatibility("hostedBy");
} else if (compatibilities.contains("notCompatible")) {
info.setCompatibility("notCompatible");
} else {
info.setCompatibility("UNKNOWN");
}
for (final String s : (String[]) rs.getArray("organizations").getArray()) {
info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim());
}
return info;
};
}
@Transactional
public void addDsAndApis(final Datasource ds, final List<ApiDetails> apis) throws DsmException {
try {
saveDs(ds);
if (apis != null) {
for (final ApiDetails api : apis) {
api.setDatasource(ds.getId());
if (StringUtils.isBlank(api.getId())) {
api.setId(createId(api));
log.info(String.format("missing api id, created '%s'", api.getId()));
}
addApi(asDbEntry(api));
log.info("API saved, id: " + api.getId());
}
}
} catch (final Throwable e) {
log.error("Error saving ds and/or api", e);
throw new DsmException("Error saving ds and/or api", e);
}
}
@Transactional
@Cacheable("brosable_terms")
public List<BrowseTerm> browseTerm(final DsmBrowsableFields f) {
switch (f) {
case type:
return simpleDsWithApisRepository.browseTermsForType();
case collectedfrom:
return simpleDsWithApisRepository.browseTermsForCollectedFrom();
case compliance:
return simpleDsWithApisRepository.browseTermsForCompliance();
case country:
return simpleDsWithApisRepository.browseTermsForCountry();
case protocol:
return simpleDsWithApisRepository.browseTermsForProtocol();
case active:
return simpleDsWithApisRepository.browseTermsForActive();
case consenttermsofuse:
return simpleDsWithApisRepository.browseTermsForConsenttermsofuse();
case fulltextdownload:
return simpleDsWithApisRepository.browseTermsForFulltextdownload();
default:
throw new RuntimeException("not implemeted");
}
}
public Page<SimpleDsWithApis> searchByField(final DsmBrowsableFields f, final String value, final int page, final int size) {
switch (f) {
case type:
return simpleDsWithApisRepository.findByType(value, PageRequest.of(page, size));
case collectedfrom:
return simpleDsWithApisRepository.findByCollectedFrom(value, PageRequest.of(page, size));
case compliance:
return simpleDsWithApisRepository.findByCompliance(value, PageRequest.of(page, size));
case country:
return simpleDsWithApisRepository.findByCountry(value, PageRequest.of(page, size));
case protocol:
return simpleDsWithApisRepository.findByProtocol(value, PageRequest.of(page, size));
case active:
return simpleDsWithApisRepository.findByActive(Boolean.valueOf(value), PageRequest.of(page, size));
case consenttermsofuse:
return simpleDsWithApisRepository.findByConsenttermsofuse(Boolean.valueOf(value), PageRequest.of(page, size));
case fulltextdownload:
return simpleDsWithApisRepository.findByFulltextdownload(Boolean.valueOf(value), PageRequest.of(page, size));
default:
throw new RuntimeException("not implemeted");
}
}
public Page<SimpleDsWithApis> search(final String value, final int page, final int size) {
return simpleDsWithApisRepository.search(value, PageRequest.of(page, size));
}
public Api findApi(final String id) throws DsmException {
return apiRepository.findById(id).orElseThrow(() -> new DsmException("Api not found. ID: " + id));
}
public void updateUpdateApiCollectionInfo(final String apiId, final String mdId, final long total) {
apiRepository.updateLastCollectionInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
}
public void updateUpdateApiAggregationInfo(final String apiId, final String mdId, final long total) {
apiRepository.updateLastAggregationInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
}
public void updateUpdateApiDownloadInfo(final String apiId, final String mdId, final long total) {
apiRepository.updateLastDownloadInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total);
}
}