new_funders_api #17
|
@ -113,6 +113,10 @@
|
||||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||||
<artifactId>jackson-datatype-joda</artifactId>
|
<artifactId>jackson-datatype-joda</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||||
|
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>joda-time</groupId>
|
<groupId>joda-time</groupId>
|
||||||
<artifactId>joda-time</artifactId>
|
<artifactId>joda-time</artifactId>
|
||||||
|
|
|
@ -1,93 +0,0 @@
|
||||||
package eu.dnetlib.openaire.funders;
|
|
||||||
|
|
||||||
import java.time.LocalDate;
|
|
||||||
import java.time.format.DateTimeFormatter;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
|
||||||
import org.springframework.data.domain.PageRequest;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
|
|
||||||
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
|
|
||||||
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
|
|
||||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
|
|
||||||
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
|
|
||||||
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
|
|
||||||
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
|
|
||||||
public class FunderDao {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private FunderRepository funderRepository;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MongoLoggerClient mongoLoggerClient;
|
|
||||||
|
|
||||||
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
|
||||||
|
|
||||||
private static final Log log = LogFactory.getLog(FunderDao.class);
|
|
||||||
|
|
||||||
public FunderDbEntry findFunder(final String funderId) throws FundersApiException {
|
|
||||||
return funderRepository.findById(funderId)
|
|
||||||
.map(this::addAggregationHistory)
|
|
||||||
.orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId));
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<FunderDbEntry> listFunders(final int page, final int size) throws FundersApiException {
|
|
||||||
return funderRepository.findAll(PageRequest.of(page, size))
|
|
||||||
.getContent()
|
|
||||||
.stream()
|
|
||||||
.map(this::addAggregationHistory)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<String> listFunderIds(final int page, final int size) throws FundersApiException {
|
|
||||||
return funderRepository.findAll(PageRequest.of(page, size))
|
|
||||||
.getContent()
|
|
||||||
.stream()
|
|
||||||
.map(FunderDbEntry::getId)
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public long count() {
|
|
||||||
return funderRepository.count();
|
|
||||||
}
|
|
||||||
|
|
||||||
private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
|
|
||||||
|
|
||||||
final List<LocalDate> dates = funder.getDatasources()
|
|
||||||
.stream()
|
|
||||||
.map(FunderDatasource::getId)
|
|
||||||
.map(id -> {
|
|
||||||
try {
|
|
||||||
return mongoLoggerClient.getAggregationHistoryV2(id);
|
|
||||||
} catch (final DsmApiException e) {
|
|
||||||
log.error("Error retrieving the aggregation history");
|
|
||||||
// throw new RuntimeException(e);
|
|
||||||
return new ArrayList<AggregationInfo>();
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.flatMap(List::stream)
|
|
||||||
.filter(AggregationInfo::isCompletedSuccessfully)
|
|
||||||
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
|
|
||||||
.map(AggregationInfo::getDate)
|
|
||||||
.distinct()
|
|
||||||
.map(s -> LocalDate.parse(s, DATEFORMATTER))
|
|
||||||
.sorted(Comparator.reverseOrder())
|
|
||||||
.toList();
|
|
||||||
|
|
||||||
funder.setAggregationDates(dates);
|
|
||||||
|
|
||||||
return funder;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -0,0 +1,139 @@
|
||||||
|
package eu.dnetlib.openaire.funders;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.FilenameFilter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import javax.annotation.PostConstruct;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||||
|
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||||
|
|
||||||
|
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
|
||||||
|
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
|
||||||
|
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
|
||||||
|
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
|
||||||
|
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
|
||||||
|
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
|
||||||
|
public class FunderService {
|
||||||
|
|
||||||
|
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private FunderRepository funderRepository;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private MongoLoggerClient mongoLoggerClient;
|
||||||
|
|
||||||
|
private File tempFile;
|
||||||
|
|
||||||
|
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||||
|
|
||||||
|
private static final Log log = LogFactory.getLog(FunderService.class);
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void init() {
|
||||||
|
for (final File f : new File("/tmp").listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
|
||||||
|
deleteFile(f);
|
||||||
|
}
|
||||||
|
|
||||||
|
new Thread(this::updateFunders).start();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteFile(final File f) {
|
||||||
|
if (f != null && f.exists()) {
|
||||||
|
log.info("Deleting file: " + f.getAbsolutePath());
|
||||||
|
f.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Scheduled(cron = "${openaire.exporter.funders.cron}")
|
||||||
|
public void updateFunders() {
|
||||||
|
try {
|
||||||
|
final ObjectMapper mapper = new ObjectMapper();
|
||||||
|
mapper.registerModule(new JavaTimeModule());
|
||||||
|
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
|
||||||
|
|
||||||
|
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, new File("/tmp"));
|
||||||
|
|
||||||
|
log.info("Generating funders file: " + tmp.getAbsolutePath());
|
||||||
|
|
||||||
|
try (final FileWriter writer = new FileWriter(tmp)) {
|
||||||
|
writer.write("[");
|
||||||
|
boolean first = true;
|
||||||
|
for (final FunderDbEntry funder : funderRepository.findAll()) {
|
||||||
|
log.info(" - adding: " + funder.getId());
|
||||||
|
// addAggregationHistory(funder);
|
||||||
|
|
||||||
|
if (first) {
|
||||||
|
first = false;
|
||||||
|
} else {
|
||||||
|
writer.write(",");
|
||||||
|
}
|
||||||
|
writer.write(mapper.writeValueAsString(funder));
|
||||||
|
}
|
||||||
|
writer.write("]");
|
||||||
|
log.info("Publish funders file: " + tmp.getAbsolutePath());
|
||||||
|
|
||||||
|
synchronized (this) {
|
||||||
|
deleteFile(tempFile);
|
||||||
|
setTempFile(tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (final IOException e) {
|
||||||
|
log.error("Error generating funders file", e);
|
||||||
|
throw new RuntimeException("Error generating funders file", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addAggregationHistory(final FunderDbEntry funder) {
|
||||||
|
|
||||||
|
final List<LocalDate> dates = funder.getDatasources()
|
||||||
|
.stream()
|
||||||
|
.map(FunderDatasource::getId)
|
||||||
|
.map(id -> {
|
||||||
|
try {
|
||||||
|
return mongoLoggerClient.getAggregationHistoryV2(id);
|
||||||
|
} catch (final DsmApiException e) {
|
||||||
|
log.error("Error retrieving the aggregation history", e);
|
||||||
|
throw new RuntimeException("Error retrieving the aggregation history", e);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.flatMap(List::stream)
|
||||||
|
.filter(AggregationInfo::isCompletedSuccessfully)
|
||||||
|
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
|
||||||
|
.map(AggregationInfo::getDate)
|
||||||
|
.distinct()
|
||||||
|
.map(s -> LocalDate.parse(s, DATEFORMATTER))
|
||||||
|
.sorted(Comparator.reverseOrder())
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
funder.setAggregationDates(dates);
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized File getTempFile() {
|
||||||
|
return tempFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTempFile(final File tempFile) {
|
||||||
|
this.tempFile = tempFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,18 +1,25 @@
|
||||||
package eu.dnetlib.openaire.funders;
|
package eu.dnetlib.openaire.funders;
|
||||||
|
|
||||||
import java.util.List;
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
import org.springframework.web.bind.annotation.CrossOrigin;
|
import org.springframework.web.bind.annotation.CrossOrigin;
|
||||||
import org.springframework.web.bind.annotation.PathVariable;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMethod;
|
import org.springframework.web.bind.annotation.RequestMethod;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
import eu.dnetlib.openaire.common.AbstractExporterController;
|
import eu.dnetlib.openaire.common.AbstractExporterController;
|
||||||
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
|
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
|
||||||
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
|
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
import io.swagger.v3.oas.annotations.responses.ApiResponse;
|
||||||
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
import io.swagger.v3.oas.annotations.responses.ApiResponses;
|
||||||
|
@ -27,60 +34,41 @@ import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
public class FundersApiController extends AbstractExporterController {
|
public class FundersApiController extends AbstractExporterController {
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private FunderDao fDao;
|
private FunderService service;
|
||||||
|
|
||||||
@RequestMapping(value = "/funders/{page}/{size}", produces = {
|
private static final Log log = LogFactory.getLog(FundersApiController.class);
|
||||||
|
|
||||||
|
@RequestMapping(value = "/funders", produces = {
|
||||||
"application/json"
|
"application/json"
|
||||||
}, method = RequestMethod.GET)
|
}, method = RequestMethod.GET)
|
||||||
@Operation(summary = "get a page of funders", description = "get a page of funders")
|
@Operation(summary = "get all funders", description = "get all funders")
|
||||||
@ApiResponses(value = {
|
@ApiResponses(value = {
|
||||||
@ApiResponse(responseCode = "200", description = "OK"),
|
@ApiResponse(responseCode = "200", description = "OK"),
|
||||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
@ApiResponse(responseCode = "500", description = "unexpected error")
|
||||||
})
|
})
|
||||||
public List<FunderDbEntry> getFunders(
|
public void getFunders(final HttpServletResponse res) throws FundersApiException {
|
||||||
@PathVariable final int page,
|
|
||||||
@PathVariable final int size) throws FundersApiException {
|
|
||||||
|
|
||||||
return fDao.listFunders(page, size);
|
res.setContentType(MediaType.APPLICATION_JSON_VALUE);
|
||||||
|
|
||||||
|
final File file = service.getTempFile();
|
||||||
|
|
||||||
|
if (file == null) {
|
||||||
|
log.error("Missing temp file (NULL)");
|
||||||
|
throw new FundersApiException("Missing temp file (NULL)");
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/funder/{id}", produces = {
|
if (!file.exists()) {
|
||||||
"application/json"
|
log.error("Missing temp file " + service.getTempFile());
|
||||||
}, method = RequestMethod.GET)
|
throw new FundersApiException("Missing temp file " + service.getTempFile());
|
||||||
@Operation(summary = "get the funder details", description = "complete funder information")
|
|
||||||
@ApiResponses(value = {
|
|
||||||
@ApiResponse(responseCode = "200", description = "OK"),
|
|
||||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
|
||||||
})
|
|
||||||
public FunderDbEntry getFunderDetails(@PathVariable final String id) throws FundersApiException {
|
|
||||||
return fDao.findFunder(id);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/funders/{page}/{size}/ids", produces = {
|
try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
|
||||||
"application/json"
|
IOUtils.copy(in, out);
|
||||||
}, method = RequestMethod.GET)
|
return;
|
||||||
@Operation(summary = "get the list of funder ids", description = "get the list of funder ids")
|
} catch (final Exception e) {
|
||||||
@ApiResponses(value = {
|
log.error("Error reading file " + service.getTempFile(), e);
|
||||||
@ApiResponse(responseCode = "200", description = "OK"),
|
throw new FundersApiException("Error reading file " + service.getTempFile(), e);
|
||||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
|
||||||
})
|
|
||||||
public List<String> listFunderIds(
|
|
||||||
@PathVariable final int page,
|
|
||||||
@PathVariable final int size) throws FundersApiException {
|
|
||||||
|
|
||||||
return fDao.listFunderIds(page, size);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@RequestMapping(value = "/funders/count", produces = {
|
|
||||||
"application/json"
|
|
||||||
}, method = RequestMethod.GET)
|
|
||||||
@Operation(summary = "count the funders", description = "count the funders")
|
|
||||||
@ApiResponses(value = {
|
|
||||||
@ApiResponse(responseCode = "200", description = "OK"),
|
|
||||||
@ApiResponse(responseCode = "500", description = "unexpected error")
|
|
||||||
})
|
|
||||||
public long countFunders() throws FundersApiException {
|
|
||||||
return fDao.count();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,3 +36,4 @@ openaire.exporter.cache.ttl = 43200000
|
||||||
|
|
||||||
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
|
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
|
||||||
|
|
||||||
|
openaire.exporter.funders.cron = 0 30 2 * * *
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
|
ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
-- TODO OCCORRE METTERE DELLE DISTINCT
|
||||||
|
|
||||||
CREATE VIEW funders_view AS SELECT
|
CREATE VIEW funders_view AS SELECT
|
||||||
o.id AS id,
|
o.id AS id,
|
||||||
o.legalshortname AS legalshortname,
|
o.legalshortname AS legalshortname,
|
||||||
|
|
|
@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
|
||||||
|
|
||||||
public class FunderContextClientTest {
|
public class FunderContextClientTest {
|
||||||
|
|
||||||
private FunderDao fDao;
|
private FunderService fDao;
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
public void setUp() {
|
public void setUp() {
|
||||||
fDao = new FunderDao();
|
fDao = new FunderService();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
Loading…
Reference in New Issue