uoa-repository-manager-service/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java

1079 lines
47 KiB
Java

package eu.dnetlib.repo.manager.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.repo.manager.domain.Repository;
import eu.dnetlib.repo.manager.domain.RepositoryInterface;
import eu.dnetlib.domain.enabling.Vocabulary;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.domain.dto.Role;
import eu.dnetlib.repo.manager.domain.dto.User;
import eu.dnetlib.repo.manager.exception.BrokerException;
import eu.dnetlib.repo.manager.exception.RepositoryServiceException;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService;
import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater;
import eu.dnetlib.repo.manager.service.security.AuthorizationService;
import eu.dnetlib.repo.manager.service.security.RoleMappingService;
import eu.dnetlib.repo.manager.utils.Converter;
import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mitre.openid.connect.model.OIDCAuthenticationToken;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Service("repositoryService")
public class RepositoryServiceImpl implements RepositoryService {
private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class);
private final AuthorizationService authorizationService;
private final RoleMappingService roleMappingService;
private final AaiRegistryService registryCalls;
private final AuthoritiesUpdater authoritiesUpdater;
private final RestTemplate restTemplate;
private final VocabularyLoader vocabularyLoader;
private final PiWikService piWikService;
private final EmailUtils emailUtils;
private final ValidatorService validatorService;
@Value("${services.provide.clients.dsm}")
private String baseAddress;
@Value("${services.provide.adminEmail}")
private String adminEmail;
@Value("${services.provide.usageStatisticsDiagramsBaseURL}")
private String usageStatisticsDiagramsBaseURL;
@Value("${services.provide.usageStatisticsNumbersBaseURL}")
private String usageStatisticsNumbersBaseURL;
private final Converter converter;
private static final Map<String, List<String>> dataSourceClass = new HashMap<>();
private static final Map<String, String> invertedDataSourceClass = new HashMap<>();
private final String[] vocabularyNames = {"dnet:countries", "dnet:eosc_datasource_types", "dnet:compatibilityLevel"};
private final Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<>();
private final Map<String, String> countriesMap = new HashMap<>();
private final Map<String, String> inverseCountriesMap = new HashMap<>();
private HttpHeaders httpHeaders;
@Autowired
public RepositoryServiceImpl(AuthorizationService authorizationService,
RoleMappingService roleMappingService,
AaiRegistryService registryCalls,
AuthoritiesUpdater authoritiesUpdater,
VocabularyLoader vocabularyLoader,
RestTemplate restTemplate,
Converter converter,
@Lazy EmailUtils emailUtils,
@Lazy ValidatorService validatorService,
@Lazy PiWikService piWikService) {
this.authorizationService = authorizationService;
this.roleMappingService = roleMappingService;
this.registryCalls = registryCalls;
this.authoritiesUpdater = authoritiesUpdater;
this.vocabularyLoader = vocabularyLoader;
this.piWikService = piWikService;
this.converter = converter;
this.emailUtils = emailUtils;
this.validatorService = validatorService;
this.restTemplate = restTemplate;
}
@PostConstruct
private void init() {
LOGGER.debug("Initialization method of repository api!");
LOGGER.debug("Updated version!");
for (String key : this.getVocabulary("dnet:datasource_typologies").getAsMap().keySet()) {
if (key.contains("aggregator")) {
dataSourceClass.putIfAbsent("aggregator", new ArrayList<>());
dataSourceClass.get("aggregator").add(key);
} else if (key.contains("crissystem")) {
dataSourceClass.putIfAbsent("dris", new ArrayList<>());
dataSourceClass.get("dris").add(key);
} else if (key.contains("pubsrepository::journal")) { // do not change order -->
dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal"));
} else if (key.contains("pubsrepository")) { // do not change order <--
dataSourceClass.putIfAbsent("opendoar", new ArrayList<>());
dataSourceClass.get("opendoar").add(key);
} else if (key.contains("datarepository")) {
dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown"));
}
}
for (Map.Entry<String, List<String>> entry : dataSourceClass.entrySet()) {
entry.getValue().forEach(v -> invertedDataSourceClass.put(v, entry.getKey()));
}
httpHeaders = new HttpHeaders();
httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8);
for (String vocName : vocabularyNames) {
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
}
Country[] countries = getCountries();
for (Country c : countries) {
countriesMap.put(c.getName(), c.getCode());
inverseCountriesMap.put(c.getCode(), c.getName());
}
}
@Override
public Country[] getCountries() {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/countries")
.build().encode();
return restTemplate.getForObject(uriComponents.toUri(), Country[].class);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<Repository> getRepositories(List<String> ids) throws JSONException {
return getRepositories(ids, 0, 10);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<Repository> getRepositories(List<String> ids, int page, int size) throws JSONException {
List<Repository> repos = new ArrayList<>();
LOGGER.debug("Retrieving repositories with ids : " + String.join(", ", ids));
UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
RequestFilter requestFilter = new RequestFilter();
for (String repoId : ids) {
requestFilter.setId(repoId);
List<Repository> rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, List.class);
// repos.addAll(converter.toRepositoryList(new JSONObject(rs)));
}
for (Repository r : repos)
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
return repos;
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids) throws Exception {
return getRepositoriesSnippets(ids, 0, 10);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids, int page, int size) throws Exception {
List<RepositorySnippet> resultSet = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
// here page should be 0
UriComponents uriComponents = searchSnipperDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
RequestFilter requestFilter = new RequestFilter();
try {
for (String repoId : ids) {
requestFilter.setId(repoId);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
}
} catch (Exception e) {
LOGGER.debug("Exception on getRepositoriesSnippetOfUser", e);
throw e;
}
LOGGER.debug("resultSet:" + resultSet);
resultSet.parallelStream().forEach(repositorySnippet -> {
repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId()));
});
return resultSet;
}
@Override
public List<RepositorySnippet> getRepositoriesByCountry(String country,
String mode,
Boolean managed) throws JSONException, IOException {
LOGGER.debug("Getting repositories by country!");
int page = 0;
int size = 100;
List<RepositorySnippet> resultSet = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
String filterKey = "UNKNOWN";
if (mode.equalsIgnoreCase("repository"))
filterKey = "Repository";
else if (mode.equalsIgnoreCase("cris"))
filterKey = "CRIS system";
LOGGER.debug("Country code equals : " + country);
LOGGER.debug("Filter mode equals : " + filterKey);
UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
RequestFilter requestFilter = new RequestFilter();
requestFilter.setCountry(country);
requestFilter.setEoscDatasourceType(filterKey);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
while (jsonArray.length() > 0) {
resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
page += 1;
uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
}
return resultSet;
}
public List<RepositorySnippet> searchRegisteredRepositories(String country, String typology, String englishName,
String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception {
LOGGER.debug("Searching registered repositories");
Paging<RepositorySnippet> snippets = null;
ObjectMapper mapper = new ObjectMapper();
UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize));
RequestFilter requestFilter = new RequestFilter();
requestFilter.setCountry(country);
requestFilter.setTypology(typology);
requestFilter.setOfficialname(officialName);
requestFilter.setEnglishname(englishName);
try {
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
if (rs == null) {
LOGGER.error(String.format("DSM response is null : [url=%s]", uriComponents.toUri()));
} else {
JSONObject response = new JSONObject(rs);
JSONArray jsonArray = (JSONArray) response.get("datasourceInfo");
Header header = mapper.readValue(response.get("header").toString(), Header.class);
snippets = Paging.of(header,
mapper.readValue(
String.valueOf(jsonArray),
mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
}
} catch (Exception e) {
LOGGER.error("Error searching registered datasources", e);
throw e;
}
return snippets != null ? snippets.getResults() : null; // TODO: return paging when ui is compatible
}
@Override
public int getTotalRegisteredRepositories() {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/countregistered")
.queryParam("fromDate", "1900-01-01")
.build().encode();
return restTemplate.getForObject(uriComponents.toUri(), Integer.class);
}
private Repository updateRepositoryInfo(Repository r) throws JSONException {
r.setInterfaces(this.getRepositoryInterface(r.getId()));
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
return r;
}
@Override
public List<Repository> getRepositoriesOfUser(String page, String size) throws JSONException {
String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail();
LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail);
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
return getRepositories(new ArrayList<>(repoIds));
}
@Override
public List<Repository> getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException {
LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail);
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));
return getRepositories(new ArrayList<>(repoIds));
}
@Override
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String page, String size) throws Exception {
return getRepositoriesSnippetsOfUser(null, page, size);
}
@Override
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception {
int from = Integer.parseInt(page) * Integer.parseInt(size);
int to = from + Integer.parseInt(size);
List<String> repoIds = new ArrayList<>();
if (userEmail != null && !"".equals(userEmail)) {
repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail)));
} else {
repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()));
}
if (repoIds.size() < from) {
return Collections.emptyList();
} else if (repoIds.size() < to) {
to = repoIds.size();
}
return getRepositoriesSnippets(repoIds.subList(from, to)); // FIXME: returns less results if some repos are not found
}
@Override
public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException {
LOGGER.debug("Retrieving repositories with id : " + id);
RepositorySnippet repo;
UriComponents uriComponents = searchSnipperDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
if (jsonArray.length() == 0)
throw new ResourceNotFoundException();
repo = converter.toRepositorySnippet(jsonArray.getJSONObject(0));
return repo;
}
@Override
public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException {
LOGGER.debug("Retrieving repositories with id : " + id);
Repository repo;
UriComponents uriComponents = searchDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
// String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
// JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
DatasourceResponse response;
response = (DatasourceResponse) restTemplate.postForObject(uriComponents.toUri(), requestFilter, DatasourceResponse.class);
List<DatasourceDetails> datasources = response.getDatasourceInfo();
if (datasources.size() == 0)
throw new ResourceNotFoundException();
// repo = converter.toRepository(jsonArray.getJSONObject(0));
// return updateRepositoryInfo(repo);
return updateRepositoryInfo(converter.toRepository(datasources.get(0)));
}
@Override
public List<AggregationDetails> getRepositoryAggregations(String id) throws JSONException {
LOGGER.debug("Retrieving aggregations for repository with id : " + id);
UriComponents uriComponents = getAggregationHistory(id);
String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo");
List<AggregationDetails> aggregationHistory = new ArrayList<>(converter.toAggregationHistory(aggregationInfo));
return aggregationHistory;
}
@Override
public List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException {
List<AggregationDetails> res = getRepositoryAggregations(id);
return res.subList(from, Math.min(from + size, res.size()));
}
@Override
public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException {
LOGGER.debug("Retrieving aggregations (by year) for repository with id : " + id);
List<AggregationDetails> aggregationHistory = getRepositoryAggregations(id);
Map<String, List<AggregationDetails>> aggregationByYear = new HashMap<>();
return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory);
}
private Map<String, List<AggregationDetails>> createYearMap(List<AggregationDetails> aggregationHistory) {
aggregationHistory = aggregationHistory.stream()
.sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
.collect(Collectors.toList());
return aggregationHistory.stream()
.collect(Collectors.groupingBy(AggregationDetails::getYear));
}
@Override
public List<Repository> getRepositoriesByName(String name,
String page,
String size) throws JSONException {
LOGGER.debug("Retrieving repositories with official name : " + name);
UriComponents uriComponents = searchDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setOfficialname(name);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
List<Repository> repos = converter.toRepositoryList(new JSONObject(rs));
for (Repository r : repos)
updateRepositoryInfo(r);
return repos;
}
@Override
public List<RepositoryInterface> getRepositoryInterface(String id) throws JSONException {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/")
.path("/{id}")
.build().expand(id).encode();
// String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
ApiDetailsResponse rs = restTemplate.getForObject(uriComponents.toUri(), ApiDetailsResponse.class);
// TODO STOP FILTERING OUT "sword", "rest" AND FIX UI!
List<ApiDetails> res = new ArrayList<>();
for (ApiDetails det: rs.getApi()) {
if (!det.getProtocol().equals("sword") &&
!det.getProtocol().equals("rest") &&
!det.getProtocol().equals("ftp")) {
res.add(det);
}
}
return converter.toRepositoryInterfaceList(res);
}
@Override
public Repository addRepository(String datatype, Repository repository) throws Exception {
LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId());
repository.setActivationId(UUID.randomUUID().toString());
repository.setCollectedfrom("infrastruct_::openaire");
// Date now = new Date();
// repository.setRegistrationdate(now);
// repository.setConsentTermsOfUseDate(now);
// repository.setLastConsentTermsOfUseDate(now);
if (datatype.equals("journal")) {
repository.setEoscDatasourceType("Journal archive");
repository.setId("openaire____::issn" + repository.getIssn());
repository.setNamespaceprefix("issn" + repository.getIssn());
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
} else if (datatype.equals("aggregator")) {
repository.setEoscDatasourceType("Aggregator");
repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialname()));
repository.setNamespaceprefix(DigestUtils.md5Hex(repository.getOfficialname()).substring(0, 12));
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
} else {
if (repository.getTypology().contains("crissystem")) {
repository.setEoscDatasourceType("CRIS system");
} else {
repository.setEoscDatasourceType("Repository");
}
this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication());
}
// TODO: move the following code elsewhere (creation and assignment of role to user) ??
// Create new role
String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId());
Role newRole = new Role(newRoleName, repository.getOfficialname());
Integer couId = null;
try {
couId = registryCalls.createRole(newRole);
} catch (HttpClientErrorException e) {
couId = registryCalls.getCouId(newRoleName);
if (couId == null) {
LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
}
} catch (Exception e) {
LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
throw e;
}
// Assign new role to the user that created it
Integer coPersonId = registryCalls.getCoPersonIdByIdentifier();
if (couId != null) {
Integer role = registryCalls.getRoleId(coPersonId, couId);
try {
registryCalls.assignMemberRole(coPersonId, couId, role);
// Add role to current user authorities
authoritiesUpdater.addRole(roleMappingService.convertRepoIdToAuthority(repository.getId()));
} catch (Exception e) {
LOGGER.debug("Exception on assign role to user during add repository", e);
throw e;
}
}
return repository;
}
/* update method acting as add -> send email with registration topic/body*/
private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/update/")
.build()
.encode();
// FIXME: problematic
// String json_repository = converter.toJson(repository);
// LOGGER.debug("JSON to add(update) -> " + json_repository);
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders); // TODO: check if it works (Repository contains extra fields)
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserRegistrationEmail(repository, authentication);
emailUtils.sendAdminRegistrationEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending email", e);
}
} else
LOGGER.error("Error storing repository: " + responseEntity.getBody().toString());
return repository;
}
@Override
public Repository updateRepository(Repository repository, Authentication authentication) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/update/")
.build()
.encode();
// FIXME: problematic
// String json_repository = converter.toJson(repository);
// LOGGER.debug("JSON to update -> " + json_repository);
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders);
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity
, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication);
emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} else
LOGGER.debug(responseEntity.getBody().toString());
return repository;
}
private void storeRepository(Repository repository, Authentication authentication) throws Exception {
Date utilDate = new Date();
Timestamp date = new Timestamp(utilDate.getTime());
repository.setDateofcollection(date);
repository.setAggregator("OPENAIRE");
// repository.setCountryCode(countriesMap.get(repository.getCountryName()));
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/add/")
.build()
.encode();
// String json_repository = converter.toJson(repository);
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders);
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserRegistrationEmail(repository, authentication);
emailUtils.sendAdminRegistrationEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} else {
LOGGER.debug(responseEntity.getBody().toString());
}
}
@Override
public void deleteRepositoryInterface(String id,
String registeredBy) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/")
.path("/{id}")
.build().expand(id).encode();
LOGGER.debug(uriComponents.toUri());
restTemplate.delete(uriComponents.toUri());
}
@Override
public RepositoryInterface addRepositoryInterface(String datatype,
String repoId,
String comment, RepositoryInterface repositoryInterface) throws Exception {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
Repository e = this.getRepositoryById(repoId);
repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype);
// String json_interface = converter.toJson(e, repositoryInterface);
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/add/")
.build()
.encode();
HttpEntity<RepositoryInterface> httpEntity = new HttpEntity<>(repositoryInterface, httpHeaders);
restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class);
try {
emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, authentication);
emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, authentication);
} catch (Exception ex) {
LOGGER.error("Error sending emails: " + ex);
}
submitInterfaceValidation(e, getAuthenticatedUser().getEmail(), repositoryInterface, false);
return repositoryInterface;
}
@Override
public RepositoryInterface updateRepositoryInterface(String repoId,
String comment,
RepositoryInterface repositoryInterface) throws Exception {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseurl());
this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompatibility());
this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());
Repository repository = this.getRepositoryById(repoId);
try {
try {
emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication);
emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication);
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} catch (Exception e) {
LOGGER.warn("Could not send emails", e);
}
submitInterfaceValidation(getRepositoryById(repoId), getAuthenticatedUser().getEmail(), repositoryInterface, true);
return repositoryInterface;
}
private User getAuthenticatedUser() {
return User.from(((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo());
}
private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting) throws ValidatorServiceException {
JobForValidation job = new JobForValidation();
job.setActivationId(UUID.randomUUID().toString());
job.setAdminEmails(Collections.singletonList(this.adminEmail));
job.setBaseUrl(iFace.getBaseurl());
job.setDatasourceId(repo.getId());
job.setDesiredCompatibilityLevel(iFace.getCompatibilityOverride());
job.setInterfaceId(iFace.getId());
job.setOfficialName(repo.getOfficialname());
job.setRepoType(repo.getEoscDatasourceType());
job.setUserEmail(userEmail);
String accessSet = iFace.getAccessSet();
job.setValidationSet((accessSet.isEmpty() ? "none" : accessSet));
job.setRecords(-1);
job.setRegistration(!updateExisting);
job.setUpdateExisting(updateExisting);
this.validatorService.submitJobForValidation(job);
}
private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) {
iFace.setDatasource(repo.getId());
iFace.setContentdescription("metadata");
iFace.setCompatibility("UNKNOWN");
if (datatype.equals("re3data"))
iFace.setAccessFormat("oai_datacite");
else
iFace.setAccessFormat("oai_dc");
// FIXME: this will probably not work
if (repo.getEoscDatasourceType() != null && !repo.getEoscDatasourceType().isEmpty())
iFace.setTypology(repo.getEoscDatasourceType());
else if (datatype.equalsIgnoreCase("journal"))
iFace.setTypology("pubsrepository::journal");
else if (datatype.equalsIgnoreCase("aggregator"))
iFace.setTypology("aggregator::pubsrepository::unknown");
else if (datatype.equalsIgnoreCase("opendoar"))
iFace.setTypology("pubsrepository::unknown");
else if (datatype.equalsIgnoreCase("re3data"))
iFace.setTypology("datarepository::unknown");
iFace.setRemovable(true);
iFace.setProtocol("oai");
iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");
iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));
if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) {
LOGGER.debug("set is empty: " + iFace.getAccessSet());
// iFace.removeAccessSet();
iFace.setAccessSet("none");
}
return iFace;
}
@Override
public List<String> getDnetCountries() {
LOGGER.debug("Getting dnet-countries!");
return converter.readFile("countries.txt");
}
@Override
public List<String> getTypologies() {
return converter.readFile("typologies.txt");
}
@Override
public List<Timezone> getTimezones() {
List<String> timezones = converter.readFile("timezones.txt");
return converter.toTimezones(timezones);
}
@Override
public List<String> getUrlsOfUserRepos(String userEmail,
String page,
String size) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/api/baseurl/")
.path("/{page}/{size}")
.build().expand(page, size).encode();
RequestFilter requestFilter = new RequestFilter();
requestFilter.setRegisteredby(userEmail);
return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class));
}
private Vocabulary getVocabulary(String vocName) {
if (!vocabularyMap.containsKey(vocName)) {
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
}
return vocabularyMap.get(vocName);
}
@Override
public Map<String, String> getCompatibilityClasses(String mode) {
LOGGER.debug("Getting compatibility classes for mode: " + mode);
Map<String, String> retMap = new HashMap<String, String>();
Map<String, String> compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap();
boolean foundData = false;
for (Map.Entry<String, String> entry : compatibilityClasses.entrySet()) {
if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL))
return compatibilityClasses;
else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) {
if (entry.getKey().matches("^openaire[1-9].0_data$")) {
retMap.put(entry.getKey(), entry.getValue());
foundData = true;
}
}
else if (mode.equalsIgnoreCase("cris")) {
if (entry.getKey().contains("openaire-cris")) {
retMap.put(entry.getKey(), entry.getValue());
foundData = true;
}
} else {
if (entry.getKey().matches("^openaire[1-9].0$") || entry.getKey().equals("driver"))
retMap.put(entry.getKey(), entry.getValue());
}
}
//TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED
if ((mode.equalsIgnoreCase("repository") || mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) && !foundData)
retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)");
return retMap;
}
@Override
public Map<String, String> getDatasourceClasses(String mode) {
LOGGER.debug("Getting datasource classes for mode: " + mode);
Map<String, String> retMap = new HashMap<String, String>();
// TODO: refactor (remove?)
for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
if (mode.equalsIgnoreCase("aggregator")) {
if (entry.getKey().contains("aggregator"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("journal")) {
if (entry.getKey().contains("journal"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("opendoar")) {
if (entry.getKey().contains("pubsrepository"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("re3data")) {
if (entry.getKey().contains("datarepository"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("dris")) {
if (entry.getKey().contains("crissystem"))
retMap.put(entry.getKey(), entry.getValue());
}
if (mode.equalsIgnoreCase("fairsharing")) {
retMap.put(entry.getKey(), entry.getValue());
}
}
if (mode.equals("fairsharing")) {
return retMap;
}
return filterResults(retMap, mode);
}
private Map<String, String> filterResults(Map<String, String> map, String mode) {
HashMap<String, String> filteredMap = new HashMap<>();
if (map != null && mode != null) {
for (String key : map.keySet())
if (dataSourceClass.get(mode).contains(key))
filteredMap.put(key, map.get(key));
return filteredMap;
}
return Collections.emptyMap();
}
@Override
public String getCountryName(String countryCode) {
return inverseCountriesMap.get(countryCode);
}
@Override
public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException {
try {
MetricsInfo metricsInfo = new MetricsInfo();
metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL);
metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId)));
return metricsInfo;
} catch (Exception e) {
LOGGER.error("Error while getting metrics info for repository: ", e);
throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR);
}
}
@Override
public Map<String, String> getListLatestUpdate(String mode) throws JSONException {
Map<String, String> dates = new HashMap<>();
if (mode.equals("repository")) {
dates.put("opendoar", converter.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate()));
dates.put("fairsharing", converter.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate()));
// create re3data last collection date
// dates.put("re3data", converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate()));
List<RepositoryInterface> re3interfaces = getRepositoryInterface("openaire____::re3data");
String re3Date = null;
for (RepositoryInterface interf : re3interfaces) {
if (interf.getLastCollectionDate() != null) {
re3Date = converter.toString(interf.getLastCollectionDate());
break;
}
}
dates.put("re3data", re3Date);
return dates;
}
else if (mode.equals("cris"))
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate()));
else if (mode.equals("opendoar")) // TODO: remove this and else clause
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
else
/*
* first api of re3data has null value on collection date
* */
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
}
private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/oaiset")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("oaiSet", validationSet)
.build().encode();
restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class);
}
private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/baseurl")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("baseUrl", baseUrl)
.build().encode();
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
}
private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/compliance")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("compliance", compliance)
.build().encode();
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
}
private MetricsNumbers getMetricsNumbers(String openAIREID) throws BrokerException {
//build the uri params
UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks");
//create new template engine
RestTemplate template = new RestTemplate();
template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());
ResponseEntity<MetricsNumbers> resp;
//communicate with endpoint
resp = template.exchange(
builder.build().encode().toUri(),
HttpMethod.GET,
null,
new ParameterizedTypeReference<MetricsNumbers>() {
});
return resp.getBody();
}
private String getOpenAIREId(String repoId) {
if (repoId != null && repoId.contains("::")) {
return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]);
}
return null;
}
private UriComponents getAggregationHistory(String repoId) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/aggregationhistory/")
.path(repoId)
.build().expand(repoId).encode();
}
private UriComponents searchDatasource(String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/searchdetails/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", "officialname")
.queryParam("order", "ASCENDING")
.build().expand(page, size).encode();
}
private UriComponents searchSnipperDatasource(String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/searchsnippet/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", "officialname")
.queryParam("order", "ASCENDING")
.build().expand(page, size).encode();
}
private UriComponents searchRegisteredDatasource(String requestSortBy, String order, String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/searchregistered/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", requestSortBy)
.queryParam("order", order)
.build().expand(page, size).encode();
}
private String getRepositoryType(String typology) {
return invertedDataSourceClass.get(typology);
}
private List<String> getRoleIdsFromUserRoles(String userEmail) {
Integer coPersonId = registryCalls.getCoPersonIdByEmail(userEmail);
JsonArray roles;
ArrayList<String> roleIds = new ArrayList<>();
ArrayList<Integer> couIds = new ArrayList<>();
if (coPersonId != null) {
roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE);
for (JsonElement role : roles) {
JsonObject object = role.getAsJsonObject();
if (object.get("CouId") == null) {
continue;
}
couIds.add(object.get("CouId").getAsInt());
}
roleIds.addAll(registryCalls.getCouNames(couIds).values());
}
return roleIds;
}
}