package eu.dnetlib.repo.manager.service;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.Repository;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.enabling.Vocabulary;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.domain.dto.Role;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService;
import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater;
import eu.dnetlib.repo.manager.service.security.AuthorizationService;
import eu.dnetlib.repo.manager.service.security.RoleMappingService;
import eu.dnetlib.repo.manager.utils.Converter;
import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.http.entity.ContentType;
import org.apache.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mitre.openid.connect.model.OIDCAuthenticationToken;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.util.MimeType;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

@Service("repositoryService")
public class RepositoryServiceImpl implements RepositoryService {

    private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class);

    private final AuthorizationService authorizationService;
    private final RoleMappingService roleMappingService;
    private final AaiRegistryService registryCalls;
    private final AuthoritiesUpdater authoritiesUpdater;
    private final RestTemplate restTemplate;
    private final VocabularyLoader vocabularyLoader;
    private final PiWikService piWikService;
    private final EmailUtils emailUtils;
    private final ValidatorService validatorService;

    @Value("${api.baseAddress}")
    private String baseAddress;

    @Value("${services.repo-manager.adminEmail}")
    private String adminEmail;

    @Value("${services.repomanager.usageStatisticsDiagramsBaseURL}")
    private String usageStatisticsDiagramsBaseURL;

    @Value("${services.repomanager.usageStatisticsNumbersBaseURL}")
    private String usageStatisticsNumbersBaseURL;


    private static final Map<String, List<String>> dataSourceClass = new HashMap<>();
    private static final Map<String, String> invertedDataSourceClass = new HashMap<>();


    private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"};
    private final Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<>();
    private final Map<String, String> countriesMap = new HashMap<>();
    private final Map<String, String> inverseCountriesMap = new HashMap<>();

    private HttpHeaders httpHeaders;

    @Autowired
    public RepositoryServiceImpl(AuthorizationService authorizationService,
                                 RoleMappingService roleMappingService,
                                 AaiRegistryService registryCalls,
                                 AuthoritiesUpdater authoritiesUpdater,
                                 VocabularyLoader vocabularyLoader,
                                 RestTemplate restTemplate,
                                 @Lazy EmailUtils emailUtils,
                                 @Lazy ValidatorService validatorService,
                                 @Lazy PiWikService piWikService) {
        this.authorizationService = authorizationService;
        this.roleMappingService = roleMappingService;
        this.registryCalls = registryCalls;
        this.authoritiesUpdater = authoritiesUpdater;
        this.vocabularyLoader = vocabularyLoader;
        this.piWikService = piWikService;
        this.emailUtils = emailUtils;
        this.validatorService = validatorService;
        this.restTemplate = restTemplate;
    }

    private String getAuthenticatedUserEmail() {
        OIDCAuthenticationToken authenticationToken = (OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
        return authenticationToken.getUserInfo().getEmail();
    }


    @PostConstruct
    private void init() {
        LOGGER.debug("Initialization method of repository api!");
        LOGGER.debug("Updated version!");

        dataSourceClass.put("opendoar", Arrays.asList("pubsrepository::institutional", "pubsrepository::thematic", "pubsrepository::unknown", "pubsrepository::mock"));
        dataSourceClass.put("re3data", Collections.singletonList("datarepository::unknown"));
        dataSourceClass.put("journal", Collections.singletonList("pubsrepository::journal"));
        dataSourceClass.put("aggregator", Arrays.asList("aggregator::pubsrepository::institutional", "aggregator::pubsrepository::journals", "aggregator::datarepository", "aggregator::pubsrepository::unknown"));

        invertedDataSourceClass.put("pubsrepository::institutional", "opendoar");
        invertedDataSourceClass.put("pubsrepository::thematic", "opendoar");
        invertedDataSourceClass.put("pubsrepository::unknown", "opendoar");
        invertedDataSourceClass.put("pubsrepository::mock", "opendoar");
        invertedDataSourceClass.put("datarepository::unknown", "re3data");
        invertedDataSourceClass.put("pubsrepository::journal", "journal");
        invertedDataSourceClass.put("aggregator::pubsrepository::institutional", "aggregator");
        invertedDataSourceClass.put("aggregator::pubsrepository::journals", "aggregator");
        invertedDataSourceClass.put("aggregator::datarepository", "aggregator");
        invertedDataSourceClass.put("aggregator::pubsrepository::unknown", "aggregator");


        httpHeaders = new HttpHeaders();
        httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8);

        for (String vocName : vocabularyNames) {
            vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
        }

        Country[] countries = getCountries();
        for (Country c : countries) {
            countriesMap.put(c.getName(), c.getCode());
            inverseCountriesMap.put(c.getCode(), c.getName());
        }


    }

    @Override
    public Country[] getCountries() {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/countries")
                .build().encode();
        return restTemplate.getForObject(uriComponents.toUri(), Country[].class);
    }
    
    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
    //  another way for paging must be implemented.
    @Override
    public List<Repository> getRepositories(List<String> ids) throws JSONException {
        return getRepositories(ids, 0, 10);
    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
    //  another way for paging must be implemented.
    @Override
    public List<Repository> getRepositories(List<String> ids, int page, int size) throws JSONException {
        List<Repository> repos = new ArrayList<>();
        LOGGER.debug("Retreiving repositories with ids : " + String.join(", ", ids));
        UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
        RequestFilter requestFilter = new RequestFilter();

        for (String repoId : ids) {
            requestFilter.setId(repoId);
            String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

            repos.addAll(Converter.jsonToRepositoryList(new JSONObject(rs)));
        }

        for (Repository r : repos)
            r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
        return repos;
    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
    //  another way for paging must be implemented.
    @Override
    public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids) throws Exception {
        return getRepositoriesSnippets(ids, 0, 10);
    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
    //  another way for paging must be implemented.
    @Override
    public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids, int page, int size) throws Exception {
        List<RepositorySnippet> resultSet = new ArrayList<>();
        ObjectMapper mapper = new ObjectMapper();

        // here page should be 0
        UriComponents uriComponents = searchSnipperDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
        RequestFilter requestFilter = new RequestFilter();

        try {
            for (String repoId : ids) {
                requestFilter.setId(repoId);

                String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
                JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
                resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
                        mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
            }
        } catch (Exception e) {
            LOGGER.debug("Exception on getRepositoriesSnippetOfUser", e);
            throw e;
        }

        LOGGER.debug("resultSet:" + resultSet);
        resultSet.parallelStream().forEach(repositorySnippet -> {
            repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId()));
        });
        return resultSet;
    }


    @Override
    public List<RepositorySnippet> getRepositoriesByCountry(String country,
                                                            String mode,
                                                            Boolean managed) throws JSONException, IOException {

        LOGGER.debug("Getting repositories by country!");
        int page = 0;
        int size = 100;
        List<RepositorySnippet> resultSet = new ArrayList<>();
        ObjectMapper mapper = new ObjectMapper();

        String filterKey = "UNKNOWN";
        if (mode.equalsIgnoreCase("opendoar"))
            filterKey = "openaire____::opendoar";
        else if (mode.equalsIgnoreCase("re3data"))
            filterKey = "openaire____::re3data";


        LOGGER.debug("Country code equals : " + country);
        LOGGER.debug("Filter mode equals : " + filterKey);

        UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setCountry(country);
        requestFilter.setCollectedfrom(filterKey);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
        while (jsonArray.length() > 0) {

LOGGER.debug("json: " + jsonArray);

            resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
                    mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
            page += 1;
            uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
            rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
            jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
        }
        return resultSet;
    }

    public List<RepositorySnippet> searchRegisteredRepositories(String country, String typology, String englishName,
                                                                String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception {

        LOGGER.debug("Searching registered repositories");

        List<RepositorySnippet> resultSet = new ArrayList<>();
        ObjectMapper mapper = new ObjectMapper();

        UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize));

        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setCountry(country);
        requestFilter.setTypology(typology);
        requestFilter.setOfficialname(officialName);
        requestFilter.setEnglishname(englishName);

        try {
            String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
            JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

            resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));

            return resultSet;
        } catch (Exception e) {
            LOGGER.error("Error searching registered datasources", e);
            throw e;
        }
    }

    @Override
    public int getTotalRegisteredRepositories() {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/countregistered")
                .queryParam("fromDate", "1900-01-01")
                .build().encode();

        return restTemplate.getForObject(uriComponents.toUri(), Integer.class);
    }

    private Repository updateRepositoryInfo(Repository r) throws JSONException {

        /*
         * from datasource class
         * we get the datasource type form the inverted map
         * */
        r.setDatasourceType(getRepositoryType(r.getDatasourceClass()));
        r.setInterfaces(this.getRepositoryInterface(r.getId()));
        r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
        r.setCountryName(getCountryName(r.getCountryCode()));
        return r;
    }


    private Collection<Repository> getRepositoriesByMode(String mode, List<Repository> rs) {

        List<Repository> reps = new ArrayList<>();
        for (Repository r : rs) {
            if (r.getCollectedFrom() != null && r.getCollectedFrom().equals(mode))
                reps.add(r);

        }
        return reps;
    }

    @Override
    public List<Repository> getRepositoriesOfUser(String page, String size) throws JSONException {
        String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail();
        LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail);
        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
        return getRepositories(new ArrayList<>(repoIds));
    }

    @Override
    public List<Repository> getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException {
        LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail);
        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));
        return getRepositories(new ArrayList<>(repoIds));
    }

    @Override
    public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String page, String size) throws Exception {
        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
        return getRepositoriesSnippets(new ArrayList<>(repoIds));
    }

    @Override
    public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception {
        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));
        return getRepositoriesSnippets(new ArrayList<>(repoIds));
    }

    @Override
    public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException {

        LOGGER.debug("Retreiving repositories with id : " + id);
        RepositorySnippet repo = null;
        UriComponents uriComponents = searchSnipperDatasource("0", "100");
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setId(id);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        if (jsonArray.length() == 0)
            throw new ResourceNotFoundException();

        repo = Converter.jsonToRepositorySnippetObject(jsonArray.getJSONObject(0));
        return repo;
    }

    @Override
    public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException {

        LOGGER.debug("Retreiving repositories with id : " + id);
        Repository repo = null;
        UriComponents uriComponents = searchDatasource("0", "100");
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setId(id);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        if (jsonArray.length() == 0)
            throw new ResourceNotFoundException();

        repo = Converter.jsonToRepositoryObject(jsonArray.getJSONObject(0));
        return updateRepositoryInfo(repo);
    }


    @Override
    public List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException {

        LOGGER.debug("Retreiving aggregations for repository with id : " + id);
        UriComponents uriComponents = searchDatasource(from + "", size + "");
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setId(id);

        List<AggregationDetails> aggregationHistory = new ArrayList<>();

        long start = System.currentTimeMillis();
        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        long end = System.currentTimeMillis();

        System.out.println("Aggregations request through rest template took " + (end - start) + "ms");
        JSONObject repository = new JSONObject(rs);

        if (repository.getJSONArray("datasourceInfo").length() == 0)
            return aggregationHistory;

        start = System.currentTimeMillis();
        aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0)));
        end = System.currentTimeMillis();

        System.out.println("Getting aggregations history from json " + (end - start) + "ms");
        return aggregationHistory.size() == 0 ? aggregationHistory : aggregationHistory.stream()
                .sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
                .limit(size)
                .collect(Collectors.toList());

    }

    @Override
    public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException {
        LOGGER.debug("Retreiving aggregations (by year) for repository with id : " + id);
        UriComponents uriComponents = searchDatasource("0", "100");
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setId(id);

        List<AggregationDetails> aggregationHistory = new ArrayList<>();
        Map<String, List<AggregationDetails>> aggregationByYear = new HashMap<>();

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        JSONObject repository = new JSONObject(rs);

        if (repository.getJSONArray("datasourceInfo").length() == 0)
            return aggregationByYear;

        aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0)));
        return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory);
    }

    private Map<String, List<AggregationDetails>> createYearMap(List<AggregationDetails> aggregationHistory) {
        Map<String, List<AggregationDetails>> aggregationByYear;
        aggregationHistory = aggregationHistory.stream()
                .sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
                .collect(Collectors.toList());

        return aggregationHistory.stream()
                .collect(Collectors.groupingBy(AggregationDetails::getYear));
    }


    @Override
    public List<Repository> getRepositoriesByName(String name,
                                                  String page,
                                                  String size) throws JSONException {

        LOGGER.debug("Retreiving  repositories with official name : " + name);
        UriComponents uriComponents = searchDatasource("0", "100");
        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setOfficialname(name);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
        List<Repository> repos = Converter.jsonToRepositoryList(new JSONObject(rs));
        for (Repository r : repos)
            updateRepositoryInfo(r);
        return repos;

    }

    @Override
    public List<RepositoryInterface> getRepositoryInterface(String id) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/")
                .path("/{id}")
                .build().expand(id).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
        return Converter.jsonToRepositoryInterfaceList(new JSONObject(rs));

    }

    @Override
    public Repository addRepository(String datatype, Repository repository) throws Exception {

        LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId());

        repository.setCountryCode(countriesMap.get(repository.getCountryName()));
        repository.setActivationId(UUID.randomUUID().toString());
        repository.setCollectedFrom("infrastruct_::openaire");

        if (datatype.equals("journal")) {
            repository.setId("openaire____::issn" + repository.getIssn());
            repository.setNamespacePrefix("issn" + repository.getIssn());
            this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
        } else if (datatype.equals("aggregator")) {
            repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialName()));
            repository.setNamespacePrefix(DigestUtils.md5Hex(repository.getOfficialName()).substring(0, 12));
            this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
        } else {
            this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication());
        }

        // TODO: move the following code elsewhere (creation and assignment of role to user) ??
        // Create new role
        String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId());
        Role newRole = new Role(newRoleName, repository.getOfficialName());
        Integer couId = null;
        try {
            couId = registryCalls.createRole(newRole);
        } catch (HttpClientErrorException e) {
            couId = registryCalls.getCouId(newRoleName);
            if (couId == null) {
                LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
            }
        } catch (Exception e) {
            LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
            throw e;
        }

        // Assign new role to the user that created it
        Integer coPersonId = registryCalls.getCoPersonIdByIdentifier();
        if (couId != null) {
            Integer role = registryCalls.getRoleId(coPersonId, couId);
            try {
                registryCalls.assignMemberRole(coPersonId, couId, role);

                // Add role to current user authorities
                authoritiesUpdater.addRole(roleMappingService.convertRepoIdToAuthority(repository.getId()));
            } catch (Exception e) {
                LOGGER.debug("Exception on assign role to user during add repository", e);
                throw e;
            }

        }


        return repository;
    }

    /* update method acting as add -> send email with registration topic/body*/
    private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/update/")
                .build()
                .encode();

        String json_repository = Converter.repositoryObjectToJson(repository);
        LOGGER.debug("JSON to add(update) -> " + json_repository);

        HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
            try {
                emailUtils.sendUserRegistrationEmail(repository, authentication);
                emailUtils.sendAdminRegistrationEmail(repository, authentication);
            } catch (Exception e) {
                LOGGER.error("Error sending email", e);
            }
        } else
            LOGGER.error("Error storing repository: " + responseEntity.getBody().toString());

        return repository;
    }

    @Override
    public Repository updateRepository(Repository repository, Authentication authentication) throws Exception {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/update/")
                .build()
                .encode();

        String json_repository = Converter.repositoryObjectToJson(repository);

        LOGGER.debug("JSON to update -> " + json_repository);

        HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity
                , ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
            try {
                emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication);
                emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication);
            } catch (Exception e) {
                LOGGER.error("Error sending emails: " + e);
            }
        } else
            LOGGER.debug(responseEntity.getBody().toString());

        return repository;
    }

    private void storeRepository(Repository repository, Authentication authentication) throws Exception {

        Date utilDate = new Date();
        Timestamp date = new Timestamp(utilDate.getTime());
        repository.setDateOfCollection(date);
        repository.setAggregator("OPENAIRE");
        repository.setCountryCode(countriesMap.get(repository.getCountryName()));

        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/add/")
                .build()
                .encode();
        String json_repository = Converter.repositoryObjectToJson(repository);
        HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
            try {
                emailUtils.sendUserRegistrationEmail(repository, authentication);
                emailUtils.sendAdminRegistrationEmail(repository, authentication);
            } catch (Exception e) {
                LOGGER.error("Error sending emails: " + e);
            }
        } else {
            LOGGER.debug(responseEntity.getBody().toString());
        }
    }

    @Override
    public void deleteRepositoryInterface(String id,
                                          String registeredBy) {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/")
                .path("/{id}")
                .build().expand(id).encode();
        LOGGER.debug(uriComponents.toUri());
        restTemplate.delete(uriComponents.toUri());
    }

    @Override
    public RepositoryInterface addRepositoryInterface(String datatype,
                                                      String repoId,
                                                      String registeredBy,
                                                      String comment, RepositoryInterface repositoryInterface) throws Exception {
        Repository e = this.getRepositoryById(repoId);
        repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype);
        String json_interface = Converter.repositoryInterfaceObjectToJson(e, repositoryInterface);

        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/add/")
                .build()
                .encode();

        HttpEntity<String> httpEntity = new HttpEntity<>(json_interface, httpHeaders);

        restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class);

        try {
            emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
            emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
        } catch (Exception ex) {
            LOGGER.error("Error sending emails: " + ex);
        }

        submitInterfaceValidation(e, registeredBy, repositoryInterface, false);

        return repositoryInterface;
    }

    @Override
    public RepositoryInterface updateRepositoryInterface(String repoId,
                                                         String registeredBy,
                                                         String comment, RepositoryInterface repositoryInterface) throws Exception {

        this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseUrl());
        this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompliance());
        this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());

        Repository repository = this.getRepositoryById(repoId);
        try {
            try {
                emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
                emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
            } catch (Exception e) {
                LOGGER.error("Error sending emails: " + e);
            }
        } catch (Exception e) {
            LOGGER.warn("Could not send emails", e);
        }

        submitInterfaceValidation(getRepositoryById(repoId), registeredBy, repositoryInterface, true);

        return repositoryInterface;
    }

    private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting) throws ValidatorServiceException {
        JobForValidation job = new JobForValidation();

        job.setActivationId(UUID.randomUUID().toString());
        job.setAdminEmails(Collections.singletonList(this.adminEmail));
        job.setBaseUrl(iFace.getBaseUrl());
        job.setDatasourceId(repo.getId());
        job.setDesiredCompatibilityLevel(iFace.getDesiredCompatibilityLevel());
        job.setInterfaceId(iFace.getId());
        job.setOfficialName(repo.getOfficialName());
        job.setRepoType(repo.getDatasourceType());
        job.setUserEmail(userEmail);
        job.setValidationSet((iFace.getAccessSet().isEmpty() ? "none" : iFace.getAccessSet()));
        job.setRecords(-1);
        job.setRegistration(!updateExisting);
        job.setUpdateExisting(updateExisting);

        this.validatorService.submitJobForValidation(job);
    }

    private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) {

        iFace.setContentDescription("metadata");
        iFace.setCompliance("UNKNOWN");

        if (datatype.equals("re3data"))
            iFace.setAccessFormat("oai_datacite");
        else
            iFace.setAccessFormat("oai_dc");


        if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty())
            iFace.setTypology(repo.getDatasourceClass());
        else if (datatype.equalsIgnoreCase("journal"))
            iFace.setTypology("pubsrepository::journal");
        else if (datatype.equalsIgnoreCase("aggregator"))
            iFace.setTypology("aggregator::pubsrepository::unknown");
        else if (datatype.equalsIgnoreCase("opendoar"))
            iFace.setTypology("pubsrepository::unknown");
        else if (datatype.equalsIgnoreCase("re3data"))
            iFace.setTypology("datarepository::unknown");

        iFace.setRemovable(true);
        iFace.setAccessProtocol("oai");
        iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");
        iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));
        if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) {
            LOGGER.debug("set is empty: " + iFace.getAccessSet());
            iFace.removeAccessSet();
            iFace.setAccessSet("none");
        }
        return iFace;
    }

    @Override
    public List<String> getDnetCountries() {
        LOGGER.debug("Getting dnet-countries!");
        return Converter.readFile("countries.txt");
    }

    @Override
    public List<String> getTypologies() {
        return Converter.readFile("typologies.txt");
    }

    @Override
    public List<Timezone> getTimezones() {
        List<String> timezones = Converter.readFile("timezones.txt");
        return Converter.toTimezones(timezones);
    }

    @Override
    public List<String> getUrlsOfUserRepos(String userEmail,
                                           String page,
                                           String size) {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/api/baseurl/")
                .path("/{page}/{size}")
                .build().expand(page, size).encode();

        RequestFilter requestFilter = new RequestFilter();
        requestFilter.setRegisteredby(userEmail);
        return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class));
    }

    @Override
    public List<String> getDatasourceVocabularies(String mode) {

        List<String> resultSet = new ArrayList<>();
        for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
            if (mode.equalsIgnoreCase("aggregator")) {
                if (entry.getKey().contains("aggregator"))
                    resultSet.add(entry.getValue());
            } else if (mode.equalsIgnoreCase("journal")) {
                if (entry.getKey().contains("journal"))
                    resultSet.add(entry.getValue());
            } else if (mode.equalsIgnoreCase("opendoar")) {
                if (entry.getKey().contains("pubsrepository"))
                    resultSet.add(entry.getValue());
            } else if (mode.equalsIgnoreCase("re3data")) {
                if (entry.getKey().contains("datarepository"))
                    resultSet.add(entry.getValue());
            }
        }


        return resultSet;
    }

    private Vocabulary getVocabulary(String vocName) {

        if (!vocabularyMap.containsKey(vocName)) {
            vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
        }
        return vocabularyMap.get(vocName);
    }


    @Override
    public Map<String, String> getCompatibilityClasses(String mode) {

        LOGGER.debug("Getting compatibility classes for mode: " + mode);
        Map<String, String> retMap = new HashMap<String, String>();

        Map<String, String> compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap();
        boolean foundData = false;
        for (Map.Entry<String, String> entry : compatibilityClasses.entrySet()) {
            if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL))
                return compatibilityClasses;
            else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) {
                if (entry.getKey().matches("^openaire[1-9].0_data$")) {
                    retMap.put(entry.getKey(), entry.getValue());
                    foundData = true;
                }
            } else {
                if (entry.getKey().matches("^openaire[1-9].0$") || entry.getKey().equals("driver"))
                    retMap.put(entry.getKey(), entry.getValue());
            }
        }

        //TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED
        if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA) && !foundData)
            retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)");

        return retMap;
    }

    @Override
    public Map<String, String> getDatasourceClasses(String mode) {

        LOGGER.debug("Getting datasource classes for mode: " + mode);

        Map<String, String> retMap = new HashMap<String, String>();

        for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
            if (mode.equalsIgnoreCase("aggregator")) {
                if (entry.getKey().contains("aggregator"))
                    retMap.put(entry.getKey(), entry.getValue());
            } else if (mode.equalsIgnoreCase("journal")) {
                if (entry.getKey().contains("journal"))
                    retMap.put(entry.getKey(), entry.getValue());
            } else if (mode.equalsIgnoreCase("opendoar")) {
                if (entry.getKey().contains("pubsrepository"))
                    retMap.put(entry.getKey(), entry.getValue());
            } else if (mode.equalsIgnoreCase("re3data")) {
                if (entry.getKey().contains("datarepository"))
                    retMap.put(entry.getKey(), entry.getValue());
            }
        }
        return filterResults(retMap, mode);

    }

    private Map<String, String> filterResults(Map<String, String> map, String mode) {

        HashMap<String, String> filteredMap = new HashMap<>();
        for (String key : map.keySet())
            if (dataSourceClass.get(mode).contains(key))
                filteredMap.put(key, map.get(key));

        return filteredMap;
    }

    @Override
    public String getCountryName(String countryCode) {
        return inverseCountriesMap.get(countryCode);
    }

    @Override
    public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException {
        try {

            MetricsInfo metricsInfo = new MetricsInfo();
            metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL);
            metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId)));
            return metricsInfo;

        } catch (Exception e) {
            LOGGER.error("Error while getting metrics info for repository: ", e);
            throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR);
        }
    }

    @Override
    public Map<String, String> getListLatestUpdate(String mode) throws JSONException {
        if (mode.equals("opendoar"))
            return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate());
        else
            /*
             * first api of re3data has null value on collection date
             * */
            return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(1).getLastCollectionDate());
    }

    private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/oaiset")
                .queryParam("dsId", repositoryId)
                .queryParam("apiId", repositoryInterfaceId)
                .queryParam("oaiSet", validationSet)
                .build().encode();
        restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class);

    }


    private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/baseurl")
                .queryParam("dsId", repositoryId)
                .queryParam("apiId", repositoryInterfaceId)
                .queryParam("baseUrl", baseUrl)
                .build().encode();
        restTemplate.postForObject(uriComponents.toUri(), null, String.class);
    }

    private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) {
        UriComponents uriComponents = UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/api/compliance")
                .queryParam("dsId", repositoryId)
                .queryParam("apiId", repositoryInterfaceId)
                .queryParam("compliance", compliance)
                .build().encode();
        restTemplate.postForObject(uriComponents.toUri(), null, String.class);
    }

    private MetricsNumbers getMetricsNumbers(String openAIREID) throws BrokerException {

        //build the uri params
        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks");

        //create new template engine
        RestTemplate template = new RestTemplate();
        template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());
        ResponseEntity<MetricsNumbers> resp;

        //communicate with endpoint
        resp = template.exchange(
                builder.build().encode().toUri(),
                HttpMethod.GET,
                null,
                new ParameterizedTypeReference<MetricsNumbers>() {
                });

        return resp.getBody();
    }

    private String getOpenAIREId(String repoId) {

        if (repoId != null && repoId.contains("::")) {
            return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]);
        }

        return null;
    }

    private UriComponents searchDatasource(String page, String size) {

        return UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/search/")
                .path("/{page}/{size}/")
                .queryParam("requestSortBy", "officialname")
                .queryParam("order", "ASCENDING")
                .build().expand(page, size).encode();
    }

    private UriComponents searchSnipperDatasource(String page, String size) {

        return UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/searchsnippet/")
                .path("/{page}/{size}/")
                .queryParam("requestSortBy", "officialname")
                .queryParam("order", "ASCENDING")
                .build().expand(page, size).encode();
    }

    private UriComponents searchRegisteredDatasource(String requestSortBy, String order, String page, String size) {

        return UriComponentsBuilder
                .fromHttpUrl(baseAddress + "/ds/searchregistered/")
                .path("/{page}/{size}/")
                .queryParam("requestSortBy", requestSortBy)
                .queryParam("order", order)
                .build().expand(page, size).encode();
    }

    private String getRepositoryType(String typology) {
        return invertedDataSourceClass.get(typology);
    }

    private List<String> getRoleIdsFromUserRoles(String userEmail) {
        Integer coPersonId = registryCalls.getCoPersonIdByEmail(userEmail);
        JsonArray roles;
        ArrayList<String> roleIds = new ArrayList<>();
        ArrayList<Integer> couIds = new ArrayList<>();
        if (coPersonId != null) {
            roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE);
            for (JsonElement role : roles) {
                JsonObject object = role.getAsJsonObject();
                if (object.get("CouId") == null) {
                    continue;
                }
                couIds.add(object.get("CouId").getAsInt());
            }

            roleIds.addAll(registryCalls.getCouNames(couIds).values());

        }
        return roleIds;
    }
}