D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new-exporter #5

Merged
michele.artini merged 21 commits from new-exporter into master 2022-04-22 10:27:22 +02:00
Conversation 0 Commits 21 Files Changed 176 +49 -38
20 changed files with 49 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b47f14aa9b - Show all commits

2
apps/dhp-broker-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-broker-public-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dhp-mdstore-manager/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
apps/dnet-orgs-database-application/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java
View File

@ -175,7 +175,7 @@ public class AdminController extends AbstractDnetController {
@GetMapping("/api/restartSuggestionsImport")
public List<String> restartSuggestionsImport(final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
importExecutor.startImport("the portal, user: " + authentication.getName());
importExecutor.startImport("the portal, user: " + UserInfo.getEmail(authentication));
return Arrays.asList("The import is running");
} else {
throw new RuntimeException("User not authorized");

4
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -1,6 +1,5 @@
package eu.dnetlib.organizations.utils;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@ -18,7 +17,6 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -436,7 +434,7 @@ public class DatabaseUtils {
@Transactional
public void importDedupEvents() throws Exception {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importDedupEvents.sql"), StandardCharsets.UTF_8));
jdbcTemplate.update("CALL import_dedup_events();");
// verifyConflictGroups(true);
}

16
apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql
View File

@ -1,4 +1,5 @@
BEGIN;
CREATE OR REPLACE PROCEDURE import_dedup_events() LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM oa_conflicts WHERE created_by = 'dedupWf' and modified_by = 'dedupWf' and reltype = 'suggested';
DELETE FROM oa_duplicates WHERE created_by = 'dedupWf' and modified_by = 'dedupWf' and reltype = 'suggested';
@ -20,7 +21,8 @@ DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
SELECT oa_original_id
FROM tmp_dedup_events
GROUP BY oa_original_id HAVING count(oa_original_id) > 1)
AND (local_id = '' OR local_id is NULL);
AND (local_id = '' OR local_id is NULL OR local_id = oa_original_id)
AND (group_id = '' OR group_id is NULL);
-- delete invalid relations (a raw org can not be suggested to multiple orgs)
DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
@ -29,6 +31,13 @@ DELETE FROM tmp_dedup_events WHERE oa_original_id IN (
GROUP BY oa_original_id HAVING count(oa_original_id) > 1)
AND local_id NOT LIKE 'openorgs____::%';
-- delete invalid groups (only one row)
DELETE FROM tmp_dedup_events WHERE group_id IN (
SELECT group_id
FROM tmp_dedup_events GROUP BY group_id
HAVING count(*) = 1
);
-- IMPORT MISSING TERMS
INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING;
@ -145,4 +154,5 @@ DELETE FROM oa_duplicates d
USING oa_duplicates d1
WHERE d.oa_original_id = d1.oa_original_id AND d.reltype = 'suggested' AND d1.reltype = 'is_similar';
COMMIT;
END;
$$;

2
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -40,7 +40,7 @@ INSERT INTO org_types(val) VALUES ('Archive'), ('Company'), ('Education'), ('Fac
UPDATE org_types SET name = val;
CREATE TABLE id_types (val text PRIMARY KEY, name text);
INSERT INTO id_types(val) VALUES ('CNRS'), ('FundRef'), ('HESA'), ('ISNI'), ('LinkedIn'), ('OrgRef'), ('UCAS'), ('UKPRN'), ('Wikidata'), ('GRID'), ('ROR');
INSERT INTO id_types(val) VALUES ('CNRS'), ('FundRef'), ('HESA'), ('ISNI'), ('LinkedIn'), ('OrgRef'), ('UCAS'), ('UKPRN'), ('Wikidata'), ('GRID'), ('ROR'), ('OrgReg');
UPDATE id_types SET name = val;
CREATE TABLE languages (val text PRIMARY KEY, name text);

6
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html
View File

@ -36,6 +36,8 @@
</ul>
</nav>
<p ng-if="mode == 'select-modal'">Click the <b>organization name</b> to add the organization (multiple selection is allowed)</p>
<table class="table table-sm table-hover" ng-if="orgs.content.length > 0">
<thead class="thead-light">
<tr class="d-flex">
@ -49,9 +51,11 @@
<tbody>
<tr ng-repeat="o in orgs.content" class="d-flex" ng-class="{'table-warning' : showStatus && o.status != 'approved'}">
<td class="col-6">
<a ng-if="mode == 'select-modal'" href="javascript:void(0)" title="{{o.id}}" ng-click="selectOrg(o)" data-dismiss="modal">{{o.name}}</a>
<a ng-if="(mode == 'select-modal') && !o.selected" href="javascript:void(0)" title="{{o.id}}" ng-click="selectOrg(o)">{{o.name}}</a>
<span ng-if="(mode == 'select-modal') && o.selected" title="{{o.id}}" class="text-muted">{{o.name}} <b>(SELECTED)</b></span>
<a ng-if="mode != 'select-modal'" href="#!/edit/0/{{o.id}}" title="{{o.id}}">{{o.name}}</a>
<span class="badge badge-warning" ng-if="showStatus && o.status != 'approved'">{{o.status}}</span>
<span ng-if="mode == 'select-modal'" class="small" ng-repeat="ourl in o.urls"><br /><b>URL: </b><a href="{{ourl}}" target="_blank" ng-if="ourl">{{ourl}}</a></span>
</td>
<td ng-class="{'col-3' : showNDups, 'col-4' : !showNDups }"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td class="col-1 text-center">{{o.acronyms.join()}}</td>

7
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html
View File

@ -6,12 +6,14 @@
<button type="button" class="close" data-dismiss="modal">&times;</button>
</div>
<div class="modal-body" style="min-height: 300px">
<form>
<div class="input-group input-group-sm">
<input type="text" class="form-control" ng-model="searchText" />
<div class="input-group-append">
<button type="button" class="btn btn-outline-primary" ng-click="search(searchText, 0, 25)">Search</button>
<button type="submit" class="btn btn-outline-primary" ng-click="search(searchText, 0, 25)">Search</button>
</div>
</div>
</form>
<div ng-show="searchValue">
<org-results-page orgs="searchOrgs"
prev-function="search(searchValue, searchOrgs.number - 1, searchOrgs.size)"
@ -22,6 +24,9 @@
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>

12
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -63,12 +63,6 @@ orgsModule.directive('selectOrgModal', function($http, $timeout) {
}
scope.selectOrg = function() {
$timeout(function() {
scope.searchText = '';
scope.searchValue = '';
scope.searchOrgs = {};
}, 1000);
if (scope.onSelect) {
scope.onSelect();
}
@ -219,6 +213,8 @@ orgsModule.directive('orgResultsPage', function($http, $location, $route) {
templateUrl: 'resources/html/parts/org_results_page.html',
link: function(scope, element, attrs, ctrl) {
scope.selectOrg = function(o) {
o.selected = true;
scope.selectedOrg.id = o.id;
scope.selectedOrg.name = o.name;
scope.selectedOrg.type = o.type;
@ -269,7 +265,7 @@ orgsModule.directive('orgDuplicates', function($http, $location, $route) {
'oaUrl' : scope.newDuplicate.urls.join(),
'oaCollectedFrom' : '',
'createdBy' : currentUser(),
'relType' : 'is_similar'
'relType' : 'suggested'
});
}
}
@ -555,7 +551,7 @@ orgsModule.controller('showEditCtrl', function ($scope, $http, $routeParams, $ro
$scope.saveDuplicates = function() {
call_http_post($http, 'api/organizations/duplicates', $scope.duplicates, function(res) {
$scope.getInfo();
alert('Events updated !!!');
alert('Duplicates updated !!!');
$scope.duplicates = res.data;
});
}

2
apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

6
apps/scholexplorer-api/pom.xml
View File

@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

2
cmd-line-apps/dhp-broker-client/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>cmd-line-apps</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
cmd-line-apps/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-broker-apps-common/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/dnet-openaire-broker-common/pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
libs/pom.xml
View File

@ -4,7 +4,7 @@
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>

2
pom.xml
View File

@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-applications</artifactId>
<version>3.2.5-SNAPSHOT</version>
<version>3.2.6-SNAPSHOT</version>
<packaging>pom</packaging>
<licenses>