144 lines
4.2 KiB
Java
144 lines
4.2 KiB
Java
package eu.dnetlib.openaire.funders;
|
|
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.FilenameFilter;
|
|
import java.io.IOException;
|
|
import java.time.LocalDate;
|
|
import java.time.format.DateTimeFormatter;
|
|
import java.util.Comparator;
|
|
import java.util.List;
|
|
import java.util.stream.Collectors;
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.fasterxml.jackson.databind.SerializationFeature;
|
|
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
|
|
|
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
|
|
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
|
|
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
|
|
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
|
|
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
|
|
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
|
|
|
@Component
|
|
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
|
|
public class FunderService {
|
|
|
|
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
|
|
|
|
@Autowired
|
|
private FunderRepository funderRepository;
|
|
|
|
@Autowired
|
|
private MongoLoggerClient mongoLoggerClient;
|
|
|
|
private File tempDir;
|
|
|
|
private File tempFile;
|
|
|
|
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
|
|
|
private static final Log log = LogFactory.getLog(FunderService.class);
|
|
|
|
@PostConstruct
|
|
public void init() {
|
|
|
|
tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));
|
|
|
|
for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
|
|
deleteFile(f);
|
|
}
|
|
|
|
new Thread(this::updateFunders).start();
|
|
}
|
|
|
|
private void deleteFile(final File f) {
|
|
if (f != null && f.exists()) {
|
|
log.info("Deleting file: " + f.getAbsolutePath());
|
|
f.delete();
|
|
}
|
|
}
|
|
|
|
@Scheduled(cron = "${openaire.exporter.funders.cron}")
|
|
public void updateFunders() {
|
|
try {
|
|
final ObjectMapper mapper = new ObjectMapper();
|
|
mapper.registerModule(new JavaTimeModule());
|
|
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
|
|
|
|
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
|
|
|
|
log.info("Generating funders file: " + tmp.getAbsolutePath());
|
|
|
|
try (final FileWriter writer = new FileWriter(tmp)) {
|
|
writer.write("[");
|
|
boolean first = true;
|
|
for (final FunderDbEntry funder : funderRepository.findAll()) {
|
|
log.info(" - adding: " + funder.getId());
|
|
addAggregationHistory(funder);
|
|
|
|
if (first) {
|
|
first = false;
|
|
} else {
|
|
writer.write(",");
|
|
}
|
|
writer.write(mapper.writeValueAsString(funder));
|
|
}
|
|
writer.write("]");
|
|
log.info("Publish funders file: " + tmp.getAbsolutePath());
|
|
|
|
deleteFile(tempFile);
|
|
setTempFile(tmp);
|
|
}
|
|
} catch (final IOException e) {
|
|
log.error("Error generating funders file", e);
|
|
throw new RuntimeException("Error generating funders file", e);
|
|
}
|
|
}
|
|
|
|
private void addAggregationHistory(final FunderDbEntry funder) {
|
|
|
|
final List<LocalDate> dates = funder.getDatasources()
|
|
.stream()
|
|
.map(FunderDatasource::getId)
|
|
.map(id -> {
|
|
try {
|
|
return mongoLoggerClient.getAggregationHistoryV2(id);
|
|
} catch (final DsmApiException e) {
|
|
log.error("Error retrieving the aggregation history", e);
|
|
throw new RuntimeException("Error retrieving the aggregation history", e);
|
|
}
|
|
})
|
|
.flatMap(List::stream)
|
|
.filter(AggregationInfo::isCompletedSuccessfully)
|
|
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
|
|
.map(AggregationInfo::getDate)
|
|
.distinct()
|
|
.map(s -> LocalDate.parse(s, DATEFORMATTER))
|
|
.sorted(Comparator.reverseOrder())
|
|
.limit(10)
|
|
.collect(Collectors.toList());
|
|
|
|
funder.setAggregationDates(dates);
|
|
}
|
|
|
|
public File getTempFile() {
|
|
return tempFile;
|
|
}
|
|
|
|
public void setTempFile(final File tempFile) {
|
|
this.tempFile = tempFile;
|
|
}
|
|
|
|
}
|