D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

import dedup events from UI

This commit is contained in:
Michele Artini 2021-09-30 09:16:06 +02:00
parent dc7dddfb37
commit c1873f27fb
7 changed files with 69 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/MainApplication.java
View File

@ -3,11 +3,9 @@ package eu.dnetlib.organizations;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
import eu.dnetlib.common.app.AbstractDnetApp;
import eu.dnetlib.organizations.importer.ImportExecution;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.service.ApiInfo;
@ -40,9 +38,4 @@ public class MainApplication extends AbstractDnetApp {
.build());
}
@Bean
public ImportExecution lastImportExecution() {
return new ImportExecution();
}
}

15
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java
View File

@ -19,6 +19,7 @@ import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.organizations.importer.ImportExecution;
import eu.dnetlib.organizations.importer.ImportExecutor;
import eu.dnetlib.organizations.model.SystemConfiguration;
import eu.dnetlib.organizations.model.utils.VocabularyTerm;
import eu.dnetlib.organizations.model.view.UserView;
@ -40,7 +41,7 @@ public class AdminController extends AbstractDnetController {
private SystemConfigurationRepository systemConfigurationRepository;
@Autowired
private ImportExecution lastImportExecution;
private ImportExecutor importExecutor;
@Autowired
private DatabaseUtils dbUtils;
@ -111,7 +112,7 @@ public class AdminController extends AbstractDnetController {
@GetMapping("/api/lastImportStatus")
private ImportExecution lastImportExecution(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
return lastImportExecution;
return importExecutor.getLastImportExecution();
} else {
throw new RuntimeException("User not authorized");
}
@ -160,4 +161,14 @@ public class AdminController extends AbstractDnetController {
throw new RuntimeException("User not authorized");
}
}
@GetMapping("/api/restartSuggestionsImport")
public List<String> restartSuggestionsImport(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
importExecutor.startImport("the portal, user: " + authentication.getName());
return Arrays.asList("The import is running");
} else {
throw new RuntimeException("User not authorized");
}
}
}

25
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java
View File

@ -15,7 +15,7 @@ import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.organizations.importer.ImportExecution;
import eu.dnetlib.organizations.importer.ImportStatus;
import eu.dnetlib.organizations.importer.ImportExecutor;
import eu.dnetlib.organizations.utils.DatabaseUtils;
@RestController
@ -29,7 +29,7 @@ public class OpenaireInternalApiController extends AbstractDnetController {
private String httpsProxy;
@Autowired
private ImportExecution lastImportExecution;
private ImportExecutor importExecutor;
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
@ -40,24 +40,7 @@ public class OpenaireInternalApiController extends AbstractDnetController {
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
}
synchronized (lastImportExecution) {
if (lastImportExecution.getStatus() != ImportStatus.RUNNING) {
lastImportExecution.startNew("Importing dedup events - request from " + req.getRemoteAddr());
new Thread(() -> {
try {
databaseUtils.importDedupEvents();
lastImportExecution.complete();
} catch (final Throwable e) {
lastImportExecution.fail(e);
}
}).start();
} else {
final long now = System.currentTimeMillis();
return new ImportExecution(null, now, now, ImportStatus.NOT_LAUNCHED, "An other import is running");
}
}
return lastImportExecution;
return importExecutor.startImport("IP " + req.getRemoteAddr());
}
@GetMapping("/import/dedupEvents/status")
@ -66,7 +49,7 @@ public class OpenaireInternalApiController extends AbstractDnetController {
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
}
return lastImportExecution;
return importExecutor.getLastImportExecution();
}
@GetMapping("/refresh/fulltextIndex")

40
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/importer/ImportExecutor.java Normal file
View File

@ -0,0 +1,40 @@
package eu.dnetlib.organizations.importer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.organizations.utils.DatabaseUtils;
@Component
public class ImportExecutor {
@Autowired
private DatabaseUtils databaseUtils;
private final ImportExecution lastImportExecution = new ImportExecution();
public ImportExecution getLastImportExecution() {
return lastImportExecution;
}
public ImportExecution startImport(final String caller) {
synchronized (lastImportExecution) {
if (lastImportExecution.getStatus() != ImportStatus.RUNNING) {
lastImportExecution.startNew("Importing dedup events - request from " + caller);
new Thread(() -> {
try {
databaseUtils.importDedupEvents();
lastImportExecution.complete();
} catch (final Throwable e) {
lastImportExecution.fail(e);
}
}).start();
} else {
final long now = System.currentTimeMillis();
return new ImportExecution(null, now, now, ImportStatus.NOT_LAUNCHED, "An other import is running");
}
}
return lastImportExecution;
}
}

4
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/lastImport.html
View File

@ -1,6 +1,6 @@
<h4>Last import of suggestions</h4>
<button class="btn btn-sm btn-primary mt-3 mb-2" ng-click="refresh()"><i class="fas fa-redo"></i> refresh</button>
<button class="btn btn-sm btn-outline-primary mt-3 mb-2" ng-click="refresh()"><i class="fas fa-redo"></i> refresh</button>
<table class="table table-sm table-striped">
<tr>
@ -21,6 +21,6 @@
</tr>
<tr>
<th>Message</th>
<td><pre style="width: 800px; font-size: 9px; overflow-x: scroll;" ng-show="lastImport.message">{{lastImport.message}}</pre></td>
<td><pre style="width: 800px; overflow-x: scroll;" ng-show="lastImport.message">{{lastImport.message}}</pre></td>
</tr>
</table>

4
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/utils.html
View File

@ -21,6 +21,10 @@
<td><button class="btn btn-sm btn-link" ng-click="verifyCountriesInSuggestions()" ng-disabled="invalidSuggestedCountriesMessage">Verify countries in suggestions</button></td>
<td>{{invalidSuggestedCountriesMessage}}</td>
</tr>
<tr>
<td><button class="btn btn-sm btn-link" ng-click="restartSuggestionsImport()" ng-disabled="suggestionImportMessage">Restart the import of the suggestions</button></td>
<td>{{suggestionImportMessage}}<span ng-show="suggestionImportMessage"> - to see the current import execution <a href="#!/lastImport">click here</a></span></td>
</tr>
</tbody>
</table>

6
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -812,6 +812,7 @@ orgsModule.controller('utilsCtrl', function ($scope, $http) {
$scope.fulltextIndexMessage = '';
$scope.consistencyCheckMessage = '';
$scope.invalidSuggestedCountriesMessage = '';
$scope.suggestionImportMessage = '';
$scope.refreshFulltextIndex = function() {
$scope.fulltextIndexMessage = '...';
@ -827,6 +828,11 @@ orgsModule.controller('utilsCtrl', function ($scope, $http) {
$scope.invalidSuggestedCountriesMessage = '...';
call_http_get($http, 'api/verifyCountriesInSuggestions', function(res) { $scope.invalidSuggestedCountriesMessage = res.data[0]; });
}
$scope.restartSuggestionsImport = function() {
$scope.suggestionImportMessage = '...';
call_http_get($http, 'api/restartSuggestionsImport', function(res) { $scope.suggestionImportMessage = res.data[0]; });
}
});