api to obtain the status of the import
This commit is contained in:
parent
5f1a46cc51
commit
764c6a3ee2
|
@ -51,7 +51,7 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter {
|
||||||
.antMatchers("/resources/**", "/webjars/**", "/metrics", "/health")
|
.antMatchers("/resources/**", "/webjars/**", "/metrics", "/health")
|
||||||
.permitAll()
|
.permitAll()
|
||||||
.antMatchers("/oa_api/**")
|
.antMatchers("/oa_api/**")
|
||||||
.hasIpAddress(openaireApiValidSubnet)
|
.permitAll()
|
||||||
.anyRequest()
|
.anyRequest()
|
||||||
.authenticated()
|
.authenticated()
|
||||||
.and()
|
.and()
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
package eu.dnetlib.organizations.controller;
|
package eu.dnetlib.organizations.controller;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
|
|
||||||
|
@ -28,19 +30,46 @@ public class OpenaireInternalApiController extends AbstractDnetController {
|
||||||
|
|
||||||
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
|
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
|
||||||
|
|
||||||
|
private final ImportExecution lastExecution = new ImportExecution();
|
||||||
|
|
||||||
@GetMapping("/import/dedupEvents")
|
@GetMapping("/import/dedupEvents")
|
||||||
public List<String> importDedupEvents(final HttpServletRequest req) {
|
public ImportExecution importDedupEvents(final HttpServletRequest req) {
|
||||||
if (req.getRemoteAddr().equals(httpsProxy)) {
|
if (req.getRemoteAddr().equals(httpsProxy)) {
|
||||||
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
}
|
}
|
||||||
new Thread(databaseUtils::importDedupEvents).start();
|
|
||||||
|
|
||||||
return Arrays.asList("Importing simrels (request from " + req.getRemoteAddr() + ") ...");
|
synchronized (lastExecution) {
|
||||||
|
if (lastExecution.getStatus() != ImportStatus.RUNNING) {
|
||||||
|
lastExecution.startNew();
|
||||||
|
new Thread(() -> {
|
||||||
|
try {
|
||||||
|
databaseUtils.importDedupEvents();
|
||||||
|
lastExecution.complete();
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
lastExecution.fail(e);
|
||||||
|
log.error("Error importing conflicts and duplicates", e);
|
||||||
|
}
|
||||||
|
}).start();
|
||||||
|
} else {
|
||||||
|
throw new RuntimeException("An import is already running");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return lastExecution;
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/import/dedupEvents/status")
|
||||||
|
public final ImportExecution statusDedupEvents(final HttpServletRequest req) {
|
||||||
|
if (req.getRemoteAddr().equals(httpsProxy)) {
|
||||||
|
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
|
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
|
}
|
||||||
|
return lastExecution;
|
||||||
}
|
}
|
||||||
|
|
||||||
@GetMapping("/refresh/fulltextIndex")
|
@GetMapping("/refresh/fulltextIndex")
|
||||||
public List<String> updateFulltextIndex(final HttpServletRequest req) {
|
public final List<String> updateFulltextIndex(final HttpServletRequest req) {
|
||||||
if (req.getRemoteAddr().equals(httpsProxy)) {
|
if (req.getRemoteAddr().equals(httpsProxy)) {
|
||||||
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
|
||||||
|
@ -49,4 +78,80 @@ public class OpenaireInternalApiController extends AbstractDnetController {
|
||||||
|
|
||||||
return Arrays.asList("Updating ...");
|
return Arrays.asList("Updating ...");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class ImportExecution {
|
||||||
|
|
||||||
|
private String id;
|
||||||
|
private Long dateStart;
|
||||||
|
private Long dateEnd;
|
||||||
|
private ImportStatus status = ImportStatus.NOT_LAUNCHED;
|
||||||
|
private String error;
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(final String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getDateStart() {
|
||||||
|
return dateStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDateStart(final Long dateStart) {
|
||||||
|
this.dateStart = dateStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getDateEnd() {
|
||||||
|
return dateEnd;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDateEnd(final Long dateEnd) {
|
||||||
|
this.dateEnd = dateEnd;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ImportStatus getStatus() {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStatus(final ImportStatus status) {
|
||||||
|
this.status = status;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getError() {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setError(final String error) {
|
||||||
|
this.error = error;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void startNew() {
|
||||||
|
setId("import-" + UUID.randomUUID());
|
||||||
|
setDateStart(new Date().getTime());
|
||||||
|
setDateEnd(null);
|
||||||
|
setStatus(ImportStatus.RUNNING);
|
||||||
|
setError(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void complete() {
|
||||||
|
setDateEnd(new Date().getTime());
|
||||||
|
setStatus(ImportStatus.SUCCESS);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void fail(final Throwable e) {
|
||||||
|
setDateEnd(new Date().getTime());
|
||||||
|
setStatus(ImportStatus.FAILED);
|
||||||
|
setError(e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ImportStatus {
|
||||||
|
SUCCESS,
|
||||||
|
FAILED,
|
||||||
|
RUNNING,
|
||||||
|
NOT_LAUNCHED
|
||||||
}
|
}
|
||||||
|
|
|
@ -364,16 +364,11 @@ public class DatabaseUtils {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void importDedupEvents() {
|
public void importDedupEvents() throws Exception {
|
||||||
try {
|
log.info("Importing conflicts and duplicates...");
|
||||||
log.info("Importing conflicts and duplicates...");
|
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql")));
|
||||||
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql")));
|
log.info("...done");
|
||||||
log.info("...done");
|
// verifyConflictGroups(true);
|
||||||
|
|
||||||
// verifyConflictGroups(true);
|
|
||||||
} catch (final Exception e) {
|
|
||||||
log.error("Error importing conflicts and duplicates", e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
|
|
|
@ -1,2 +1,6 @@
|
||||||
-- TO RESOLVE MANUALLY MANAGING CONFLICTS
|
-- TO RESOLVE MANUALLY MANAGING CONFLICTS
|
||||||
select d.oa_original_id, array_agg(o.id), array_agg(o.name) from oa_duplicates d left outer join organizations o on (o.id = d.local_id) where d.reltype='is_similar' group by oa_original_id having count(reltype) > 1 and count (distinct o.status) = 1;
|
select d.oa_original_id, array_agg(o.id), array_agg(o.name) from oa_duplicates d left outer join organizations o on (o.id = d.local_id) where d.reltype='is_similar' group by oa_original_id having count(reltype) > 1 and count (distinct o.status) = 1;
|
||||||
|
|
||||||
|
|
||||||
|
-- TO force the status of the orgs
|
||||||
|
-- select name from organizations WHERE country='IT' and type='UNKNOWN' and lower(reverse(split_part(reverse(trim(name)), ' ', 1))) in ('spa', 'snc', 'srl', 'scarl') and name not ilike '%universit%' and name not ilike '%ospedale%' and name not ilike '%hospital%';
|
||||||
|
|
Loading…
Reference in New Issue