D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

status import page and system utilities

This commit is contained in:
Michele Artini 2021-09-29 12:06:17 +02:00
parent 654d3fc397
commit e25e90819b
12 changed files with 229 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MainApplication.java
View File

@ -3,9 +3,11 @@ package eu.dnetlib.organizations;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import eu.dnetlib.organizations.importer.ImportExecution;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
@ -38,4 +40,9 @@ public class MainApplication extends AbstractDnetApp {
.build());
}
@Bean
public ImportExecution lastImportExecution() {
return new ImportExecution();
}
}

33
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java
View File

@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.organizations.importer.ImportExecution;
import eu.dnetlib.organizations.model.SystemConfiguration;
import eu.dnetlib.organizations.model.utils.VocabularyTerm;
import eu.dnetlib.organizations.model.view.UserView;
@ -37,6 +38,9 @@ public class AdminController extends AbstractDnetController {
@Autowired
private SystemConfigurationRepository systemConfigurationRepository;
@Autowired
private ImportExecution lastImportExecution;
@Autowired
private DatabaseUtils dbUtils;
@ -103,6 +107,15 @@ public class AdminController extends AbstractDnetController {
}
}
@GetMapping("/api/lastImportStatus")
private ImportExecution lastImportExecution(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
return lastImportExecution;
} else {
throw new RuntimeException("User not authorized");
}
}
@PostMapping("/api/sysconf")
public SystemConfiguration saveSysConf(@RequestBody final SystemConfiguration sysConf, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
@ -112,4 +125,24 @@ public class AdminController extends AbstractDnetController {
throw new RuntimeException("User not authorized");
}
}
@GetMapping("/api/refreshFulltextIndex")
public List<String> refreshFulltextIndex(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
new Thread(dbUtils::updateFulltextIndex).start();
return Arrays.asList("The index update is in progress, please wait a few minutes");
} else {
throw new RuntimeException("User not authorized");
}
}
@GetMapping("/api/performConsistencyCheck")
public List<String> performConsistencyCheck(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
new Thread(dbUtils::verifyConsistency).start();
return Arrays.asList("The check is running, please wait a few minutes");
} else {
throw new RuntimeException("User not authorized");
}
}
}

120
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java
View File

@ -1,10 +1,7 @@
package eu.dnetlib.organizations.controller;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.servlet.http.HttpServletRequest;
@ -17,6 +14,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.organizations.importer.ImportExecution;
import eu.dnetlib.organizations.importer.ImportStatus;
import eu.dnetlib.organizations.utils.DatabaseUtils;
@RestController
@ -29,9 +28,10 @@ public class OpenaireInternalApiController extends AbstractDnetController {
@Value("${openaire.api.https.proxy}")
private String httpsProxy;
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
@Autowired
private ImportExecution lastImportExecution;
private final ImportExecution lastExecution = new ImportExecution();
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
@GetMapping("/import/dedupEvents")
public ImportExecution importDedupEvents(final HttpServletRequest req) {
@ -40,15 +40,15 @@ public class OpenaireInternalApiController extends AbstractDnetController {
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
}
synchronized (lastExecution) {
if (lastExecution.getStatus() != ImportStatus.RUNNING) {
lastExecution.startNew("Importing dedup events - request from " + req.getRemoteAddr());
synchronized (lastImportExecution) {
if (lastImportExecution.getStatus() != ImportStatus.RUNNING) {
lastImportExecution.startNew("Importing dedup events - request from " + req.getRemoteAddr());
new Thread(() -> {
try {
databaseUtils.importDedupEvents();
lastExecution.complete();
lastImportExecution.complete();
} catch (final Throwable e) {
lastExecution.fail(e);
lastImportExecution.fail(e);
}
}).start();
} else {
@ -57,7 +57,7 @@ public class OpenaireInternalApiController extends AbstractDnetController {
}
}
return lastExecution;
return lastImportExecution;
}
@GetMapping("/import/dedupEvents/status")
@ -66,7 +66,7 @@ public class OpenaireInternalApiController extends AbstractDnetController {
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
}
return lastExecution;
return lastImportExecution;
}
@GetMapping("/refresh/fulltextIndex")
@ -80,100 +80,4 @@ public class OpenaireInternalApiController extends AbstractDnetController {
return Arrays.asList("Updating ...");
}
class ImportExecution {
private String id;
private Long dateStart;
private Long dateEnd;
private ImportStatus status = ImportStatus.NOT_YET_STARTED;
private String message;
public ImportExecution() {}
public ImportExecution(final String id, final Long dateStart, final Long dateEnd, final ImportStatus status, final String message) {
this.id = id;
this.dateStart = dateStart;
this.dateEnd = dateEnd;
this.status = status;
this.message = message;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public Long getDateStart() {
return dateStart;
}
public void setDateStart(final Long dateStart) {
this.dateStart = dateStart;
}
public Long getDateEnd() {
return dateEnd;
}
public void setDateEnd(final Long dateEnd) {
this.dateEnd = dateEnd;
}
public ImportStatus getStatus() {
return status;
}
public void setStatus(final ImportStatus status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
public void startNew(final String message) {
setId("import-" + UUID.randomUUID());
setDateStart(System.currentTimeMillis());
setDateEnd(null);
setStatus(ImportStatus.RUNNING);
setMessage(message);
log.info(message);
}
public void complete() {
setDateEnd(System.currentTimeMillis());
setStatus(ImportStatus.SUCCESS);
final long millis = getDateEnd() - getDateStart();
setMessage(String
.format("Import of dedup events completed in %d min, %d sec", TimeUnit.MILLISECONDS.toMinutes(millis), TimeUnit.MILLISECONDS.toSeconds(millis) -
TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))));
log.info(getMessage());
}
public void fail(final Throwable e) {
setDateEnd(new Date().getTime());
setStatus(ImportStatus.FAILED);
setMessage(e.getMessage());
log.error("Error importing conflicts and duplicates", e);
}
}
}
enum ImportStatus {
SUCCESS,
FAILED,
RUNNING,
NOT_LAUNCHED,
NOT_YET_STARTED
}

101
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/importer/ImportExecution.java Normal file
View File

@ -0,0 +1,101 @@
package eu.dnetlib.organizations.importer;
import java.util.Date;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.dnetlib.organizations.controller.OpenaireInternalApiController;
public class ImportExecution {
private String id;
private Long dateStart;
private Long dateEnd;
private ImportStatus status = ImportStatus.NOT_YET_STARTED;
private String message;
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
public ImportExecution() {}
public ImportExecution(final String id, final Long dateStart, final Long dateEnd, final ImportStatus status, final String message) {
this.id = id;
this.dateStart = dateStart;
this.dateEnd = dateEnd;
this.status = status;
this.message = message;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public Long getDateStart() {
return dateStart;
}
public void setDateStart(final Long dateStart) {
this.dateStart = dateStart;
}
public Long getDateEnd() {
return dateEnd;
}
public void setDateEnd(final Long dateEnd) {
this.dateEnd = dateEnd;
}
public ImportStatus getStatus() {
return status;
}
public void setStatus(final ImportStatus status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
public void startNew(final String message) {
setId("import-" + UUID.randomUUID());
setDateStart(System.currentTimeMillis());
setDateEnd(null);
setStatus(ImportStatus.RUNNING);
setMessage(message);
log.info(message);
}
public void complete() {
setDateEnd(System.currentTimeMillis());
setStatus(ImportStatus.SUCCESS);
final long millis = getDateEnd() - getDateStart();
setMessage(String
.format("Import of dedup events completed in %d min, %d sec", TimeUnit.MILLISECONDS.toMinutes(millis), TimeUnit.MILLISECONDS.toSeconds(millis) -
TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))));
log.info(getMessage());
}
public void fail(final Throwable e) {
setDateEnd(new Date().getTime());
setStatus(ImportStatus.FAILED);
setMessage(e.getMessage());
log.error("Error importing conflicts and duplicates", e);
}
}

9
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/importer/ImportStatus.java Normal file
View File

@ -0,0 +1,9 @@
package eu.dnetlib.organizations.importer;
public enum ImportStatus {
SUCCESS,
FAILED,
RUNNING,
NOT_LAUNCHED,
NOT_YET_STARTED
}

3
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.organizations.utils;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@ -407,7 +408,7 @@ public class DatabaseUtils {
@Transactional
public void importDedupEvents() throws Exception {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql")));
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql"), StandardCharsets.UTF_8));
// verifyConflictGroups(true);
}

4
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -29,11 +29,11 @@ DROP SEQUENCE IF EXISTS organizations_id_seq;
CREATE TABLE sysconf (
id text PRIMARY KEY DEFAULT 'default',
title text NOT NULL DEFAULT 'OpenOrgs Database',
title text NOT NULL,
homepage_msg text NOT NULL DEFAULT '',
readonly boolean NOT NULL DEFAULT false
);
INSERT INTO sysconf(id) VALUES ('default');
INSERT INTO sysconf(id, title) VALUES ('default', 'OpenOrgs Database');
CREATE TABLE org_types (val text PRIMARY KEY, name text);
INSERT INTO org_types(val) VALUES ('Archive'), ('Company'), ('Education'), ('Facility'), ('Government'), ('Healthcare'), ('Nonprofit'), ('Other'), ('UNKNOWN');

26
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/lastImport.html Normal file
View File

@ -0,0 +1,26 @@
<h4>Last import of suggestions</h4>
<button class="btn btn-sm btn-primary mt-3 mb-2" ng-click="refresh()"><i class="fas fa-redo"></i> refresh</button>
<table class="table table-sm table-striped">
<tr>
<th style="width: 20%">Execution ID</th>
<td style="width: 80%">{{lastImport.id}}</td>
</tr>
<tr>
<th>Execution status</th>
<td>{{lastImport.status}}</td>
</tr>
<tr>
<th>Started at</th>
<td>{{lastImport.dateStart | date:"MMM dd, yyyy 'at' HH:mm"}}</td>
</tr>
<tr>
<th>Finished at</th>
<td>{{lastImport.dateEnd | date:"MMM dd, yyyy 'at' HH:mm"}}</td>
</tr>
<tr>
<th>Message</th>
<td><pre style="width: 800px; font-size: 9px; overflow-x: scroll;" ng-show="lastImport.message">{{lastImport.message}}</pre></td>
</tr>
</table>

1
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/sysConf.html
View File

@ -30,3 +30,4 @@
</form>
</div>
</div>

6
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/utils.html Normal file
View File

@ -0,0 +1,6 @@
<h4>System utilities</h4>
<br />
<button class="btn btn-sm btn-primary" ng-click="refreshFulltextIndex()">Rebuild fulltext index</button>
<button class="btn btn-sm btn-primary" ng-click="performConsistencyCheck()">Perform consistency check</button>

29
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -381,6 +381,8 @@ orgsModule.config(function($routeProvider) {
.when('/conflicts/:country', { templateUrl: 'resources/html/pages/advanced/conflicts.html', controller: 'conflictsCtrl' })
.when('/users', { templateUrl: 'resources/html/pages/admin/users.html', controller: 'usersCtrl' })
.when('/sysconf', { templateUrl: 'resources/html/pages/admin/sysConf.html', controller: 'sysConfCtrl' })
.when('/utils', { templateUrl: 'resources/html/pages/admin/utils.html', controller: 'utilsCtrl' })
.when('/lastImport', { templateUrl: 'resources/html/pages/admin/lastImport.html', controller: 'lastImportCtrl' })
.otherwise({ redirectTo: '/search' });
});
@ -781,7 +783,7 @@ orgsModule.controller('conflictsCtrl', function ($scope, $http, $routeParams, $l
orgsModule.controller('sysConfCtrl', function ($scope, $http, $timeout, $route) {
$scope.sysconf = {};
call_http_get($http, 'api/sysconf', function(res) { $scope.sysconf = res.data; });
$scope.saveConf = function() {
@ -795,7 +797,30 @@ orgsModule.controller('sysConfCtrl', function ($scope, $http, $timeout, $route)
});
orgsModule.controller('usersCtrl', function ($scope, $http, $timeout, $route, vocabulariesService) {
orgsModule.controller('lastImportCtrl', function ($scope, $http) {
$scope.lastImport = {};
$scope.refresh = function() {
call_http_get($http, 'api/lastImportStatus', function(res) { $scope.lastImport = res.data; });
}
$scope.refresh();
});
orgsModule.controller('utilsCtrl', function ($scope, $http) {
$scope.refreshFulltextIndex = function() {
call_http_get($http, 'api/refreshFulltextIndex', function(res) { alert(res.data[0]); });
}
$scope.performConsistencyCheck = function() {
call_http_get($http, 'api/performConsistencyCheck', function(res) { alert(res.data[0]); });
}
});
orgsModule.controller('usersCtrl', function ($scope, $http, vocabulariesService) {
$scope.users = [];
$scope.vocs = {};
$scope.currentUser = {};

4
apps/dnet-orgs-database-application/src/main/resources/templates/main.html
View File

@ -98,7 +98,7 @@ fieldset > legend { font-size : 1.2rem !important; }
<advanced-menu-item menu="Organizations with new duplicates" description="Organizations that presents duplicates suggested by the Dedup OpenAIRE Workflow" url="#!/duplicates/_" badge="{{info.data.total.nDuplicates}}" first="1"></advanced-menu-item>
</div>
</li>
<li class="nav-item dropdown" sec:authorize="hasRole('ROLE_OPENORGS_USER')">
<li class="nav-item dropdown" sec:authorize="hasRole('ROLE_OPENORGS_USER')">
<a class="nav-link dropdown-toggle" href="javascript:void(0)" data-toggle="dropdown">Suggest</a>
<div class="dropdown-menu">
<a class="dropdown-item" href="#!/new">A new organization</a>
@ -110,6 +110,8 @@ fieldset > legend { font-size : 1.2rem !important; }
<a class="nav-link dropdown-toggle" href="javascript:void(0)" data-toggle="dropdown"><i class="fa fa-cog"></i></a>
<div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item" href="#!/sysconf" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')">System configuration</a>
<a class="dropdown-item" href="#!/utils" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')">System utilities</a>
<a class="dropdown-item" href="#!/lastImport" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')">Last import of suggestions</a>
<div class="dropdown-divider" sec:authorize="hasRole('ROLE_OPENORGS_ADMIN')"></div>
<a class="dropdown-item" href="#!/users">Manage users</a>
</div>