From 1bfeac01a1a2e0a003a1676517010d8a0515f52b Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 29 Oct 2020 13:18:11 +0100 Subject: [PATCH] conflicts --- .../controller/OrganizationController.java | 2 - .../model/OpenaireDuplicate.java | 5 ++ .../OpenaireConflictRepository.java | 8 ++- .../organizations/utils/DatabaseUtils.java | 66 ++++--------------- .../main/resources/sql/importDedupEvents.sql | 27 ++++++-- .../html/pages/advanced/conflicts.html | 6 +- .../resources/html/pages/search/browse.html | 2 +- .../src/test/java/SimpleTest.java | 5 ++ 8 files changed, 54 insertions(+), 67 deletions(-) create mode 100644 apps/dnet-orgs-database-application/src/test/java/SimpleTest.java diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java index 6345958c..c9d99822 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java @@ -165,8 +165,6 @@ public class OrganizationController { @GetMapping("/conflicts/byCountry/{country}") public Collection> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) { - // databaseUtils.verifyConflictGroups(false); - if (UserInfo.isSuperAdmin(authentication)) { return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream()); } else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) { diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java index 10b7129f..de1c4989 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java @@ -61,4 +61,9 @@ public class OpenaireDuplicate implements Serializable { this.relType = relType; } + @Override + public String toString() { + return String.format("OpenaireDuplicate [localId=%s, oaOriginalId=%s, relType=%s]", localId, oaOriginalId, relType); + } + } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java index 87f4939c..84cccfa6 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java @@ -22,9 +22,13 @@ public interface OpenaireConflictRepository extends JpaRepository 0) { - * - * log.info("Recreating conflicts group..."); - * - * openaireConflictRepository.resetGroupIds(); - * - * final Map> groups = new HashMap<>(); for (final OpenaireConflict w : openaireConflictRepository.findAll()) { - * final List list = findExistingGroupsForRel(w, groups); if (list.isEmpty()) { final String idGroup = generateGroupId(); - * groups.put(idGroup, new HashSet<>()); addToGroup(groups, idGroup, w); } else if (list.size() == 1) { addToGroup(groups, list.get(0), - * w); } else { final String idGroup = generateGroupId(); groups.put(idGroup, new TreeSet<>()); list.forEach(id -> - * groups.get(idGroup).addAll(groups.get(id))); list.forEach(id -> groups.remove(id)); addToGroup(groups, idGroup, w); } } - * - * for (final Entry> e : groups.entrySet()) { final String gid = e.getKey(); for (final String orgId : e.getValue()) - * { for (final OpenaireConflict oc : openaireConflictRepository.findById1AndGroupIsNull(orgId)) { oc.setGroup(gid); - * openaireConflictRepository.save(oc); } for (final OpenaireConflict oc : openaireConflictRepository.findById2AndGroupIsNull(orgId)) { - * oc.setGroup(gid); openaireConflictRepository.save(oc); } } } - * - * log.info("...conflict groups recreated"); } } - * - * private String generateGroupId() { return "group::" + UUID.randomUUID(); } - */ - - private List findExistingGroupsForRel(final OpenaireConflict w, final Map> groups) { - return groups.entrySet() - .stream() - .filter(e -> { - return e.getValue().contains(w.getId1()) || e.getValue().contains(w.getId2()); - }) - .map(e -> e.getKey()) - .distinct() - .collect(Collectors.toList()); - } - - private void addToGroup(final Map> groups, final String gid, final OpenaireConflict w) { - groups.get(gid).add(w.getId1()); - groups.get(gid).add(w.getId2()); - } - @Transactional public List makeRelation(final String id1, final String id2, final RelationType type) { final Relationship r1 = new Relationship(id1, id2, type.toString()); @@ -385,10 +343,10 @@ public class DatabaseUtils { } @Transactional - public String fixConflict(final List ids, final String user) { + public String fixConflict(final List similarIds, final String user) { final List views = - ids.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); + similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); // I create a new org final OrganizationView newOrg = new OrganizationView(); @@ -417,22 +375,26 @@ public class DatabaseUtils { final String masterId = insertOrUpdateOrganization(newOrg, user, false); // I hide the merged organizations - ids.forEach(id -> hideConflictOrgs(masterId, id)); + similarIds.forEach(id -> hideConflictOrgs(masterId, id)); - // I reassign the duplicated to the new org - final List newDuplicates = ids.stream() + // I reassign the duplicates to the new org + final List newDuplicates = similarIds.stream() .map(openaireDuplicateRepository::findByLocalId) .flatMap(l -> l.stream()) - .filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString())) .map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType())) .collect(Collectors.toList()); - newDuplicates.forEach(d -> d.setLocalId(masterId)); + + openaireDuplicateRepository.saveAll(newDuplicates); final OffsetDateTime now = OffsetDateTime.now(); - for (int i = 0; i < ids.size(); i++) { - for (int j = i + 1; j < ids.size(); j++) { - openaireConflictRepository.updateStatus(ids.get(i), ids.get(j), SimilarityType.is_similar.toString(), user, now); + for (final String similarId : similarIds) { + openaireConflictRepository.updateMultipleStatusAndResetGroup(similarId, SimilarityType.is_different.toString(), user, now); + } + + for (int i = 0; i < similarIds.size(); i++) { + for (int j = i + 1; j < similarIds.size(); j++) { + openaireConflictRepository.updateStatusAndResetGroup(similarIds.get(i), similarIds.get(j), SimilarityType.is_similar.toString(), user, now); } } diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql index 9df9e8dd..c06f2da9 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql @@ -11,13 +11,13 @@ UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_c INSERT INTO organizations(id, name, country, status, created_by, modified_by) SELECT oa_original_id, oa_name, oa_country, 'suggested', 'dedupWf', 'dedupWf' FROM tmp_dedup_events -WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id = local_id +WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND (oa_original_id = local_id OR local_id = '' OR local_id IS NULL) ON CONFLICT DO NOTHING; INSERT INTO organizations(id, name, country, status, created_by, modified_by) SELECT oa_original_id, oa_name, oa_country, 'duplicate', 'dedupWf', 'dedupWf' FROM tmp_dedup_events -WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id != local_id +WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id is NOT NULL AND local_id != '' AND local_id != oa_original_id ON CONFLICT DO NOTHING; @@ -32,12 +32,25 @@ WHERE local_id IS NOT NULL AND local_id != '' AND oa_original_id NOT LIKE 'openo ON CONFLICT DO NOTHING; --- CONFLICTS -INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by) -SELECT local_id, oa_original_id, group_id, 'dedupWf', 'dedupWf' -FROM tmp_dedup_events -WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != '' +-- CONFLICTS (I generate all the couples) +CREATE TEMPORARY TABLE tmp_conflict_groups AS + SELECT DISTINCT group_id as gid, local_id oid from tmp_dedup_events + WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != '' +UNION + SELECT DISTINCT group_id as gid, oa_original_id oid from tmp_dedup_events + WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''; + +INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by) SELECT DISTINCT + c1.oid, c2.oid, c1.gid, 'dedupWf', 'dedupWf' +FROM + tmp_conflict_groups c1 + JOIN tmp_conflict_groups c2 ON (c1.gid = c2.gid) +WHERE + c1.oid != c2.oid ON CONFLICT DO NOTHING; + + + COMMIT; diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html index 24205bfd..ccb6a72d 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html @@ -1,8 +1,6 @@

Conflicts

-

No suggestions

- -
+
Country: @@ -19,6 +17,8 @@
+
No conflicts
+
Group {{$index+1}}
diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html index 3e9d5065..64fe3048 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html @@ -14,7 +14,7 @@ - + diff --git a/apps/dnet-orgs-database-application/src/test/java/SimpleTest.java b/apps/dnet-orgs-database-application/src/test/java/SimpleTest.java new file mode 100644 index 00000000..f4bd4d71 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/test/java/SimpleTest.java @@ -0,0 +1,5 @@ + + +public class SimpleTest { + +}
# approved # suggested # deleted# duplicated# duplicate # discarded # hidden