dnet-applications/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java

package eu.dnetlib.openaire.funders;

import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;

import javax.annotation.PostConstruct;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;

import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderPid;

@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderService {

	private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
	private static final String SEPARATOR = "@=@";

	@Autowired
	private FunderRepository funderRepository;

	@Autowired
	private MongoLoggerClient mongoLoggerClient;

	private File tempDir;

	private File tempFile;

	private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");

	private static final Log log = LogFactory.getLog(FunderService.class);

	@PostConstruct
	public void init() {

		tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));

		for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
			deleteFile(f);
		}

		new Thread(this::updateFunders).start();
	}

	private void deleteFile(final File f) {
		if (f != null && f.exists()) {
			log.info("Deleting file: " + f.getAbsolutePath());
			f.delete();
		}
	}

	@Scheduled(cron = "${openaire.exporter.funders.cron}")
	public void updateFunders() {
		try {
			final ObjectMapper mapper = new ObjectMapper();
			mapper.registerModule(new JavaTimeModule());
			mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);

			final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);

			log.info("Generating funders file: " + tmp.getAbsolutePath());

			try (final FileWriter writer = new FileWriter(tmp)) {
				writer.write("[");
				boolean first = true;
				for (final FunderDbEntry funder : funderRepository.findAll()) {
					log.info("  - adding: " + funder.getId());

					// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)

					final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
							.filter(Objects::nonNull)
							.map(s -> s.split(SEPARATOR))
							.filter(arr -> arr.length == 3)
							.map(arr -> {
								final FunderDatasource ds = new FunderDatasource();
								ds.setId(arr[0].trim());
								ds.setName(arr[1].trim());
								ds.setType(arr[2].trim());
								return ds;
							})
							.filter(ds -> StringUtils.isNotBlank(ds.getId()))
							.collect(Collectors.toList());

					funder.setDatasources(datasources);

					final List<FunderPid> pids = Arrays.stream(funder.getPidsPostgres())
							.filter(Objects::nonNull)
							.map(s -> s.split(SEPARATOR))
							.filter(arr -> arr.length == 2)
							.map(arr -> {
								final FunderPid pid = new FunderPid();
								pid.setType(arr[0].trim());
								pid.setValue(arr[1].trim());
								return pid;
							})
							.filter(pid -> StringUtils.isNotBlank(pid.getValue()))
							.collect(Collectors.toList());

					funder.setPids(pids);

					// END PATCH

					addAggregationHistory(funder);

					if (first) {
						first = false;
					} else {
						writer.write(",");
					}
					writer.write(mapper.writeValueAsString(funder));
				}
				writer.write("]");
				log.info("Publish funders file: " + tmp.getAbsolutePath());

				deleteFile(tempFile);
				setTempFile(tmp);
			}
		} catch (final Throwable e) {
			log.error("Error generating funders file", e);
			throw new RuntimeException("Error generating funders file", e);
		}
	}

	private void addAggregationHistory(final FunderDbEntry funder) {

		final List<LocalDate> dates = funder.getDatasources()
				.stream()
				.map(FunderDatasource::getId)
				.map(id -> {
					try {
						return mongoLoggerClient.getAggregationHistoryV2(id);
					} catch (final DsmApiException e) {
						log.error("Error retrieving the aggregation history", e);
						throw new RuntimeException("Error retrieving the aggregation history", e);
					}
				})
				.flatMap(List::stream)
				.filter(AggregationInfo::isCompletedSuccessfully)
				.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
				.map(AggregationInfo::getDate)
				.distinct()
				.map(s -> LocalDate.parse(s, DATEFORMATTER))
				.sorted(Comparator.reverseOrder())
				.limit(10)
				.collect(Collectors.toList());

		funder.setAggregationDates(dates);
	}

	public File getTempFile() {
		return tempFile;
	}

	public void setTempFile(final File tempFile) {
		this.tempFile = tempFile;
	}

}