D-Net
/
dnet-applications
23
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactoring using temp file

This commit is contained in:
Michele Artini 2024-02-19 16:06:34 +01:00
parent fc750633b9
commit e5a9ee0b21
7 changed files with 183 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/dnet-exporter-api/pom.xml
View File

@ -113,6 +113,10 @@
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-joda</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>

93
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java
View File

@ -1,93 +0,0 @@
package eu.dnetlib.openaire.funders;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Component;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderDao {
@Autowired
private FunderRepository funderRepository;
@Autowired
private MongoLoggerClient mongoLoggerClient;
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Log log = LogFactory.getLog(FunderDao.class);
public FunderDbEntry findFunder(final String funderId) throws FundersApiException {
return funderRepository.findById(funderId)
.map(this::addAggregationHistory)
.orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId));
}
public List<FunderDbEntry> listFunders(final int page, final int size) throws FundersApiException {
return funderRepository.findAll(PageRequest.of(page, size))
.getContent()
.stream()
.map(this::addAggregationHistory)
.collect(Collectors.toList());
}
public List<String> listFunderIds(final int page, final int size) throws FundersApiException {
return funderRepository.findAll(PageRequest.of(page, size))
.getContent()
.stream()
.map(FunderDbEntry::getId)
.collect(Collectors.toList());
}
public long count() {
return funderRepository.count();
}
private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
final List<LocalDate> dates = funder.getDatasources()
.stream()
.map(FunderDatasource::getId)
.map(id -> {
try {
return mongoLoggerClient.getAggregationHistoryV2(id);
} catch (final DsmApiException e) {
log.error("Error retrieving the aggregation history");
// throw new RuntimeException(e);
return new ArrayList<AggregationInfo>();
}
})
.flatMap(List::stream)
.filter(AggregationInfo::isCompletedSuccessfully)
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
.map(AggregationInfo::getDate)
.distinct()
.map(s -> LocalDate.parse(s, DATEFORMATTER))
.sorted(Comparator.reverseOrder())
.toList();
funder.setAggregationDates(dates);
return funder;
}
}

139
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java Normal file
View File

@ -0,0 +1,139 @@
package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderService {
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
@Autowired
private FunderRepository funderRepository;
@Autowired
private MongoLoggerClient mongoLoggerClient;
private File tempFile;
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Log log = LogFactory.getLog(FunderService.class);
@PostConstruct
public void init() {
for (final File f : new File("/tmp").listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
deleteFile(f);
}
new Thread(this::updateFunders).start();
}
private void deleteFile(final File f) {
if (f != null && f.exists()) {
log.info("Deleting file: " + f.getAbsolutePath());
f.delete();
}
}
@Scheduled(cron = "${openaire.exporter.funders.cron}")
public void updateFunders() {
try {
final ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new JavaTimeModule());
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, new File("/tmp"));
log.info("Generating funders file: " + tmp.getAbsolutePath());
try (final FileWriter writer = new FileWriter(tmp)) {
writer.write("[");
boolean first = true;
for (final FunderDbEntry funder : funderRepository.findAll()) {
log.info(" - adding: " + funder.getId());
// addAggregationHistory(funder);
if (first) {
first = false;
} else {
writer.write(",");
}
writer.write(mapper.writeValueAsString(funder));
}
writer.write("]");
log.info("Publish funders file: " + tmp.getAbsolutePath());
synchronized (this) {
deleteFile(tempFile);
setTempFile(tmp);
}
}
} catch (final IOException e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
}
private void addAggregationHistory(final FunderDbEntry funder) {
final List<LocalDate> dates = funder.getDatasources()
.stream()
.map(FunderDatasource::getId)
.map(id -> {
try {
return mongoLoggerClient.getAggregationHistoryV2(id);
} catch (final DsmApiException e) {
log.error("Error retrieving the aggregation history", e);
throw new RuntimeException("Error retrieving the aggregation history", e);
}
})
.flatMap(List::stream)
.filter(AggregationInfo::isCompletedSuccessfully)
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
.map(AggregationInfo::getDate)
.distinct()
.map(s -> LocalDate.parse(s, DATEFORMATTER))
.sorted(Comparator.reverseOrder())
.collect(Collectors.toList());
funder.setAggregationDates(dates);
}
public synchronized File getTempFile() {
return tempFile;
}
public void setTempFile(final File tempFile) {
this.tempFile = tempFile;
}
}

78
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java
View File

@ -1,18 +1,25 @@
package eu.dnetlib.openaire.funders;
import java.util.List;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
@ -27,60 +34,41 @@ import io.swagger.v3.oas.annotations.tags.Tag;
public class FundersApiController extends AbstractExporterController {
@Autowired
private FunderDao fDao;
private FunderService service;
@RequestMapping(value = "/funders/{page}/{size}", produces = {
private static final Log log = LogFactory.getLog(FundersApiController.class);
@RequestMapping(value = "/funders", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get a page of funders", description = "get a page of funders")
@Operation(summary = "get all funders", description = "get all funders")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<FunderDbEntry> getFunders(
@PathVariable final int page,
@PathVariable final int size) throws FundersApiException {
public void getFunders(final HttpServletResponse res) throws FundersApiException {
return fDao.listFunders(page, size);
}
res.setContentType(MediaType.APPLICATION_JSON_VALUE);
@RequestMapping(value = "/funder/{id}", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the funder details", description = "complete funder information")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public FunderDbEntry getFunderDetails(@PathVariable final String id) throws FundersApiException {
return fDao.findFunder(id);
}
final File file = service.getTempFile();
@RequestMapping(value = "/funders/{page}/{size}/ids", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get the list of funder ids", description = "get the list of funder ids")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public List<String> listFunderIds(
@PathVariable final int page,
@PathVariable final int size) throws FundersApiException {
if (file == null) {
log.error("Missing temp file (NULL)");
throw new FundersApiException("Missing temp file (NULL)");
}
return fDao.listFunderIds(page, size);
}
if (!file.exists()) {
log.error("Missing temp file " + service.getTempFile());
throw new FundersApiException("Missing temp file " + service.getTempFile());
}
@RequestMapping(value = "/funders/count", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "count the funders", description = "count the funders")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public long countFunders() throws FundersApiException {
return fDao.count();
try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
IOUtils.copy(in, out);
return;
} catch (final Exception e) {
log.error("Error reading file " + service.getTempFile(), e);
throw new FundersApiException("Error reading file " + service.getTempFile(), e);
}
}
}

1
apps/dnet-exporter-api/src/main/resources/application.properties
View File

@ -36,3 +36,4 @@ openaire.exporter.cache.ttl = 43200000
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml
openaire.exporter.funders.cron = 0 30 2 * * *

4
apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql
View File

@ -1,5 +1,9 @@
ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
-- TODO OCCORRE METTERE DELLE DISTINCT
CREATE VIEW funders_view AS SELECT
o.id AS id,
o.legalshortname AS legalshortname,

4
apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java
View File

@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails;
public class FunderContextClientTest {
private FunderDao fDao;
private FunderService fDao;
@BeforeEach
public void setUp() {
fDao = new FunderDao();
fDao = new FunderService();
}
@Test