new_funders_api #17
|
@ -113,6 +113,10 @@
|
|||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-joda</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
|
|
|
@ -1,93 +0,0 @@
|
|||
package eu.dnetlib.openaire.funders;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
|
||||
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
|
||||
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
|
||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
|
||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
|
||||
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
|
||||
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
|
||||
public class FunderDao {
|
||||
|
||||
@Autowired
|
||||
private FunderRepository funderRepository;
|
||||
|
||||
@Autowired
|
||||
private MongoLoggerClient mongoLoggerClient;
|
||||
|
||||
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
|
||||
private static final Log log = LogFactory.getLog(FunderDao.class);
|
||||
|
||||
public FunderDbEntry findFunder(final String funderId) throws FundersApiException {
|
||||
return funderRepository.findById(funderId)
|
||||
.map(this::addAggregationHistory)
|
||||
.orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId));
|
||||
}
|
||||
|
||||
public List<FunderDbEntry> listFunders(final int page, final int size) throws FundersApiException {
|
||||
return funderRepository.findAll(PageRequest.of(page, size))
|
||||
.getContent()
|
||||
.stream()
|
||||
.map(this::addAggregationHistory)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<String> listFunderIds(final int page, final int size) throws FundersApiException {
|
||||
return funderRepository.findAll(PageRequest.of(page, size))
|
||||
.getContent()
|
||||
.stream()
|
||||
.map(FunderDbEntry::getId)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public long count() {
|
||||
return funderRepository.count();
|
||||
}
|
||||
|
||||
private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
|
||||
|
||||
final List<LocalDate> dates = funder.getDatasources()
|
||||
.stream()
|
||||
.map(FunderDatasource::getId)
|
||||
.map(id -> {
|
||||
try {
|
||||
return mongoLoggerClient.getAggregationHistoryV2(id);
|
||||
} catch (final DsmApiException e) {
|
||||
log.error("Error retrieving the aggregation history");
|
||||
// throw new RuntimeException(e);
|
||||
return new ArrayList<AggregationInfo>();
|
||||
}
|
||||
})
|
||||
.flatMap(List::stream)
|
||||
.filter(AggregationInfo::isCompletedSuccessfully)
|
||||
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
|
||||
.map(AggregationInfo::getDate)
|
||||
.distinct()
|
||||
.map(s -> LocalDate.parse(s, DATEFORMATTER))
|
||||
.sorted(Comparator.reverseOrder())
|
||||
.toList();
|
||||
|
||||
funder.setAggregationDates(dates);
|
||||
|
||||
return funder;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,139 @@
|
|||
package eu.dnetlib.openaire.funders;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
|
||||
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
|
||||
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
|
||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
|
||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
|
||||
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
|
||||
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
||||
|
||||
@Component
|
||||
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
|
||||
public class FunderService {
|
||||
|
||||
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
|
||||
|
||||
@Autowired
|
||||
private FunderRepository funderRepository;
|
||||
|
||||
@Autowired
|
||||
private MongoLoggerClient mongoLoggerClient;
|
||||
|
||||
private File tempFile;
|
||||
|
||||
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
|
||||
private static final Log log = LogFactory.getLog(FunderService.class);
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
for (final File f : new File("/tmp").listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
|
||||
deleteFile(f);
|
||||
}
|
||||
|
||||
new Thread(this::updateFunders).start();
|
||||
}
|
||||
|
||||
private void deleteFile(final File f) {
|
||||
if (f != null && f.exists()) {
|
||||
log.info("Deleting file: " + f.getAbsolutePath());
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Scheduled(cron = "${openaire.exporter.funders.cron}")
|
||||
public void updateFunders() {
|
||||
try {
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.registerModule(new JavaTimeModule());
|
||||
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
|
||||
|
||||
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, new File("/tmp"));
|
||||
|
||||
log.info("Generating funders file: " + tmp.getAbsolutePath());
|
||||
|
||||
try (final FileWriter writer = new FileWriter(tmp)) {
|
||||
writer.write("[");
|
||||
boolean first = true;
|
||||
for (final FunderDbEntry funder : funderRepository.findAll()) {
|
||||
log.info(" - adding: " + funder.getId());
|
||||
// addAggregationHistory(funder);
|
||||
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
writer.write(",");
|
||||
}
|
||||
writer.write(mapper.writeValueAsString(funder));
|
||||
}
|
||||
writer.write("]");
|
||||
log.info("Publish funders file: " + tmp.getAbsolutePath());
|
||||
|
||||
synchronized (this) {
|
||||
deleteFile(tempFile);
|
||||
setTempFile(tmp);
|
||||
}
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
log.error("Error generating funders file", e);
|
||||
throw new RuntimeException("Error generating funders file", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void addAggregationHistory(final FunderDbEntry funder) {
|
||||
|
||||
final List<LocalDate> dates = funder.getDatasources()
|
||||
.stream()
|
||||
.map(FunderDatasource::getId)
|
||||
.map(id -> {
|
||||
try {
|
||||
return mongoLoggerClient.getAggregationHistoryV2(id);
|
||||
} catch (final DsmApiException e) {
|
||||
log.error("Error retrieving the aggregation history", e);
|
||||
throw new RuntimeException("Error retrieving the aggregation history", e);
|
||||
}
|
||||
})
|
||||
.flatMap(List::stream)
|
||||
.filter(AggregationInfo::isCompletedSuccessfully)
|
||||
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
|
||||
.map(AggregationInfo::getDate)
|
||||
.distinct()
|
||||
.map(s -> LocalDate.parse(s, DATEFORMATTER))
|
||||
.sorted(Comparator.reverseOrder())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
funder.setAggregationDates(dates);
|
||||
}
|
||||
|
||||
public synchronized File getTempFile() {
|
||||
return tempFile;
|
||||
}
|
||||
|
||||
public void setTempFile(final File tempFile) {
|
||||
this.tempFile = tempFile;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,18 +1,25 @@
|
|||
package eu.dnetlib.openaire.funders;
|
||||
|
||||
import java.util.List;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.CrossOrigin;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMethod;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.openaire.common.AbstractExporterController;
|
||||
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
|
||||
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||
|
@ -27,60 +34,41 @@ import io.swagger.v3.oas.annotations.tags.Tag;
|
|||
public class FundersApiController extends AbstractExporterController {
|
||||
|
||||
@Autowired
|
||||
private FunderDao fDao;
|
||||
private FunderService service;
|
||||
|
||||
@RequestMapping(value = "/funders/{page}/{size}", produces = {
|
||||
private static final Log log = LogFactory.getLog(FundersApiController.class);
|
||||
|
||||
@RequestMapping(value = "/funders", produces = {
|
||||
"application/json"
|
||||
}, method = RequestMethod.GET)
|
||||
@Operation(summary = "get a page of funders", description = "get a page of funders")
|
||||
@Operation(summary = "get all funders", description = "get all funders")
|
||||
@ApiResponses(value = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
||||
})
|
||||
public List<FunderDbEntry> getFunders(
|
||||
@PathVariable final int page,
|
||||
@PathVariable final int size) throws FundersApiException {
|
||||
public void getFunders(final HttpServletResponse res) throws FundersApiException {
|
||||
|
||||
return fDao.listFunders(page, size);
|
||||
res.setContentType(MediaType.APPLICATION_JSON_VALUE);
|
||||
|
||||
final File file = service.getTempFile();
|
||||
|
||||
if (file == null) {
|
||||
log.error("Missing temp file (NULL)");
|
||||
throw new FundersApiException("Missing temp file (NULL)");
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/funder/{id}", produces = {
|
||||
"application/json"
|
||||
}, method = RequestMethod.GET)
|
||||
@Operation(summary = "get the funder details", description = "complete funder information")
|
||||
@ApiResponses(value = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
||||
})
|
||||
public FunderDbEntry getFunderDetails(@PathVariable final String id) throws FundersApiException {
|
||||
return fDao.findFunder(id);
|
||||
if (!file.exists()) {
|
||||
log.error("Missing temp file " + service.getTempFile());
|
||||
throw new FundersApiException("Missing temp file " + service.getTempFile());
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/funders/{page}/{size}/ids", produces = {
|
||||
"application/json"
|
||||
}, method = RequestMethod.GET)
|
||||
@Operation(summary = "get the list of funder ids", description = "get the list of funder ids")
|
||||
@ApiResponses(value = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
||||
})
|
||||
public List<String> listFunderIds(
|
||||
@PathVariable final int page,
|
||||
@PathVariable final int size) throws FundersApiException {
|
||||
|
||||
return fDao.listFunderIds(page, size);
|
||||
try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
|
||||
IOUtils.copy(in, out);
|
||||
return;
|
||||
} catch (final Exception e) {
|
||||
log.error("Error reading file " + service.getTempFile(), e);
|
||||
throw new FundersApiException("Error reading file " + service.getTempFile(), e);
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/funders/count", produces = {
|
||||
"application/json"
|
||||
}, method = RequestMethod.GET)
|
||||
@Operation(summary = "count the funders", description = "count the funders")
|
||||
@ApiResponses(value = {
|
||||
@ApiResponse(responseCode = "200", description = "OK"),
|
||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
||||
})
|
||||
public long countFunders() throws FundersApiException {
|
||||
return fDao.count();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -36,3 +36,4 @@ openaire.exporter.cache.ttl = 43200000
|
|||
|
||||
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
|
||||
|
||||
openaire.exporter.funders.cron = 0 30 2 * * *
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
|
||||
|
||||
|
||||
|
||||
-- TODO OCCORRE METTERE DELLE DISTINCT
|
||||
|
||||
CREATE VIEW funders_view AS SELECT
|
||||
o.id AS id,
|
||||
o.legalshortname AS legalshortname,
|
||||
|
|
|
@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
|
|||
|
||||
public class FunderContextClientTest {
|
||||
|
||||
private FunderDao fDao;
|
||||
private FunderService fDao;
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
fDao = new FunderDao();
|
||||
fDao = new FunderService();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
Loading…
Reference in New Issue