From 764c6a3ee24c3032ddf269bb06f50d0934576659 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 22 Apr 2021 15:42:35 +0200 Subject: [PATCH] api to obtain the status of the import --- .../organizations/MockSecurityConfig.java | 2 +- .../OpenaireInternalApiController.java | 113 +++++++++++++++++- .../organizations/utils/DatabaseUtils.java | 15 +-- .../resources/sql/someConsistencyQueries.sql | 4 + 4 files changed, 119 insertions(+), 15 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java index b8448608..e66a3422 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MockSecurityConfig.java @@ -51,7 +51,7 @@ public class MockSecurityConfig extends WebSecurityConfigurerAdapter { .antMatchers("/resources/**", "/webjars/**", "/metrics", "/health") .permitAll() .antMatchers("/oa_api/**") - .hasIpAddress(openaireApiValidSubnet) + .permitAll() .anyRequest() .authenticated() .and() diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java index ba4a3dcb..76177e48 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java @@ -1,7 +1,9 @@ package eu.dnetlib.organizations.controller; import java.util.Arrays; +import java.util.Date; import java.util.List; +import java.util.UUID; import javax.servlet.http.HttpServletRequest; @@ -28,19 +30,46 @@ public class OpenaireInternalApiController extends AbstractDnetController { private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class); + private final ImportExecution lastExecution = new ImportExecution(); + @GetMapping("/import/dedupEvents") - public List importDedupEvents(final HttpServletRequest req) { + public ImportExecution importDedupEvents(final HttpServletRequest req) { if (req.getRemoteAddr().equals(httpsProxy)) { log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); } - new Thread(databaseUtils::importDedupEvents).start(); - return Arrays.asList("Importing simrels (request from " + req.getRemoteAddr() + ") ..."); + synchronized (lastExecution) { + if (lastExecution.getStatus() != ImportStatus.RUNNING) { + lastExecution.startNew(); + new Thread(() -> { + try { + databaseUtils.importDedupEvents(); + lastExecution.complete(); + } catch (final Throwable e) { + lastExecution.fail(e); + log.error("Error importing conflicts and duplicates", e); + } + }).start(); + } else { + throw new RuntimeException("An import is already running"); + } + + } + return lastExecution; + } + + @GetMapping("/import/dedupEvents/status") + public final ImportExecution statusDedupEvents(final HttpServletRequest req) { + if (req.getRemoteAddr().equals(httpsProxy)) { + log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); + throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); + } + return lastExecution; } @GetMapping("/refresh/fulltextIndex") - public List updateFulltextIndex(final HttpServletRequest req) { + public final List updateFulltextIndex(final HttpServletRequest req) { if (req.getRemoteAddr().equals(httpsProxy)) { log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); @@ -49,4 +78,80 @@ public class OpenaireInternalApiController extends AbstractDnetController { return Arrays.asList("Updating ..."); } + + class ImportExecution { + + private String id; + private Long dateStart; + private Long dateEnd; + private ImportStatus status = ImportStatus.NOT_LAUNCHED; + private String error; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public Long getDateStart() { + return dateStart; + } + + public void setDateStart(final Long dateStart) { + this.dateStart = dateStart; + } + + public Long getDateEnd() { + return dateEnd; + } + + public void setDateEnd(final Long dateEnd) { + this.dateEnd = dateEnd; + } + + public ImportStatus getStatus() { + return status; + } + + public void setStatus(final ImportStatus status) { + this.status = status; + } + + public String getError() { + return error; + } + + public void setError(final String error) { + this.error = error; + } + + public void startNew() { + setId("import-" + UUID.randomUUID()); + setDateStart(new Date().getTime()); + setDateEnd(null); + setStatus(ImportStatus.RUNNING); + setError(null); + } + + public void complete() { + setDateEnd(new Date().getTime()); + setStatus(ImportStatus.SUCCESS); + } + + public void fail(final Throwable e) { + setDateEnd(new Date().getTime()); + setStatus(ImportStatus.FAILED); + setError(e.getMessage()); + } + + } +} + +enum ImportStatus { + SUCCESS, + FAILED, + RUNNING, + NOT_LAUNCHED } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index 55ac5f28..1100f434 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -364,16 +364,11 @@ public class DatabaseUtils { } @Transactional - public void importDedupEvents() { - try { - log.info("Importing conflicts and duplicates..."); - jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql"))); - log.info("...done"); - - // verifyConflictGroups(true); - } catch (final Exception e) { - log.error("Error importing conflicts and duplicates", e); - } + public void importDedupEvents() throws Exception { + log.info("Importing conflicts and duplicates..."); + jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql"))); + log.info("...done"); + // verifyConflictGroups(true); } @Transactional diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/someConsistencyQueries.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/someConsistencyQueries.sql index 89ee3fec..647ae3c7 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/someConsistencyQueries.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/someConsistencyQueries.sql @@ -1,2 +1,6 @@ -- TO RESOLVE MANUALLY MANAGING CONFLICTS select d.oa_original_id, array_agg(o.id), array_agg(o.name) from oa_duplicates d left outer join organizations o on (o.id = d.local_id) where d.reltype='is_similar' group by oa_original_id having count(reltype) > 1 and count (distinct o.status) = 1; + + +-- TO force the status of the orgs +-- select name from organizations WHERE country='IT' and type='UNKNOWN' and lower(reverse(split_part(reverse(trim(name)), ' ', 1))) in ('spa', 'snc', 'srl', 'scarl') and name not ilike '%universit%' and name not ilike '%ospedale%' and name not ilike '%hospital%';