D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

conflicts

This commit is contained in:
Michele Artini 2020-10-29 13:18:11 +01:00
parent 0555abf02f
commit 02f7f674ce
8 changed files with 54 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
View File

@ -165,8 +165,6 @@ public class OrganizationController {
@GetMapping("/conflicts/byCountry/{country}")
public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
// databaseUtils.verifyConflictGroups(false);
if (UserInfo.isSuperAdmin(authentication)) {
return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream());
} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {

5
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java
View File

@ -61,4 +61,9 @@ public class OpenaireDuplicate implements Serializable {
this.relType = relType;
}
@Override
public String toString() {
return String.format("OpenaireDuplicate [localId=%s, oaOriginalId=%s, relType=%s]", localId, oaOriginalId, relType);
}
}

8
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java
View File

@ -22,9 +22,13 @@ public interface OpenaireConflictRepository extends JpaRepository<OpenaireConfli
void resetGroupIds();
@Modifying
@Query(value = "update oa_conflicts set reltype = ?3, modified_by = ?4, modification_date = ?5 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
void updateStatus(String id1, String id2, String status, String user, OffsetDateTime now);
@Query(value = "update oa_conflicts set idgroup = null, reltype = ?3, modified_by = ?4, modification_date = ?5 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
void updateStatusAndResetGroup(String id1, String id2, String status, String user, OffsetDateTime now);
long countByGroupNull();
@Modifying
@Query(value = "update oa_conflicts set idgroup = null, reltype = ?2, modified_by = ?3, modification_date = ?4 where id1 = ?1 or id2 = ?1", nativeQuery = true)
void updateMultipleStatusAndResetGroup(String id, String type, String user, OffsetDateTime now);
}

66
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.RequestBody;
import eu.dnetlib.organizations.controller.UserRole;
import eu.dnetlib.organizations.model.Acronym;
import eu.dnetlib.organizations.model.OpenaireConflict;
import eu.dnetlib.organizations.model.OpenaireConflictPK;
import eu.dnetlib.organizations.model.OpenaireDuplicate;
import eu.dnetlib.organizations.model.Organization;
@ -243,47 +242,6 @@ public class DatabaseUtils {
}
}
/*
* @Transactional public void verifyConflictGroups(final boolean forceUpdate) {
*
* if (forceUpdate || openaireConflictRepository.countByGroupNull() > 0) {
*
* log.info("Recreating conflicts group...");
*
* openaireConflictRepository.resetGroupIds();
*
* final Map<String, Set<String>> groups = new HashMap<>(); for (final OpenaireConflict w : openaireConflictRepository.findAll()) {
* final List<String> list = findExistingGroupsForRel(w, groups); if (list.isEmpty()) { final String idGroup = generateGroupId();
* groups.put(idGroup, new HashSet<>()); addToGroup(groups, idGroup, w); } else if (list.size() == 1) { addToGroup(groups, list.get(0),
* w); } else { final String idGroup = generateGroupId(); groups.put(idGroup, new TreeSet<>()); list.forEach(id ->
* groups.get(idGroup).addAll(groups.get(id))); list.forEach(id -> groups.remove(id)); addToGroup(groups, idGroup, w); } }
*
* for (final Entry<String, Set<String>> e : groups.entrySet()) { final String gid = e.getKey(); for (final String orgId : e.getValue())
* { for (final OpenaireConflict oc : openaireConflictRepository.findById1AndGroupIsNull(orgId)) { oc.setGroup(gid);
* openaireConflictRepository.save(oc); } for (final OpenaireConflict oc : openaireConflictRepository.findById2AndGroupIsNull(orgId)) {
* oc.setGroup(gid); openaireConflictRepository.save(oc); } } }
*
* log.info("...conflict groups recreated"); } }
*
* private String generateGroupId() { return "group::" + UUID.randomUUID(); }
*/
private List<String> findExistingGroupsForRel(final OpenaireConflict w, final Map<String, Set<String>> groups) {
return groups.entrySet()
.stream()
.filter(e -> {
return e.getValue().contains(w.getId1()) || e.getValue().contains(w.getId2());
})
.map(e -> e.getKey())
.distinct()
.collect(Collectors.toList());
}
private void addToGroup(final Map<String, Set<String>> groups, final String gid, final OpenaireConflict w) {
groups.get(gid).add(w.getId1());
groups.get(gid).add(w.getId2());
}
@Transactional
public List<Relationship> makeRelation(final String id1, final String id2, final RelationType type) {
final Relationship r1 = new Relationship(id1, id2, type.toString());
@ -385,10 +343,10 @@ public class DatabaseUtils {
}
@Transactional
public String fixConflict(final List<String> ids, final String user) {
public String fixConflict(final List<String> similarIds, final String user) {
final List<OrganizationView> views =
ids.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
// I create a new org
final OrganizationView newOrg = new OrganizationView();
@ -417,22 +375,26 @@ public class DatabaseUtils {
final String masterId = insertOrUpdateOrganization(newOrg, user, false);
// I hide the merged organizations
ids.forEach(id -> hideConflictOrgs(masterId, id));
similarIds.forEach(id -> hideConflictOrgs(masterId, id));
// I reassign the duplicated to the new org
final List<OpenaireDuplicate> newDuplicates = ids.stream()
// I reassign the duplicates to the new org
final List<OpenaireDuplicate> newDuplicates = similarIds.stream()
.map(openaireDuplicateRepository::findByLocalId)
.flatMap(l -> l.stream())
.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType()))
.collect(Collectors.toList());
newDuplicates.forEach(d -> d.setLocalId(masterId));
openaireDuplicateRepository.saveAll(newDuplicates);
final OffsetDateTime now = OffsetDateTime.now();
for (int i = 0; i < ids.size(); i++) {
for (int j = i + 1; j < ids.size(); j++) {
openaireConflictRepository.updateStatus(ids.get(i), ids.get(j), SimilarityType.is_similar.toString(), user, now);
for (final String similarId : similarIds) {
openaireConflictRepository.updateMultipleStatusAndResetGroup(similarId, SimilarityType.is_different.toString(), user, now);
}
for (int i = 0; i < similarIds.size(); i++) {
for (int j = i + 1; j < similarIds.size(); j++) {
openaireConflictRepository.updateStatusAndResetGroup(similarIds.get(i), similarIds.get(j), SimilarityType.is_similar.toString(), user, now);
}
}

27
apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql
View File

@ -11,13 +11,13 @@ UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_c
INSERT INTO organizations(id, name, country, status, created_by, modified_by)
SELECT oa_original_id, oa_name, oa_country, 'suggested', 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id = local_id
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND (oa_original_id = local_id OR local_id = '' OR local_id IS NULL)
ON CONFLICT DO NOTHING;
INSERT INTO organizations(id, name, country, status, created_by, modified_by)
SELECT oa_original_id, oa_name, oa_country, 'duplicate', 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id != local_id
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id is NOT NULL AND local_id != '' AND local_id != oa_original_id
ON CONFLICT DO NOTHING;
@ -32,12 +32,25 @@ WHERE local_id IS NOT NULL AND local_id != '' AND oa_original_id NOT LIKE 'openo
ON CONFLICT DO NOTHING;
-- CONFLICTS
INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by)
SELECT local_id, oa_original_id, group_id, 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''
-- CONFLICTS (I generate all the couples)
CREATE TEMPORARY TABLE tmp_conflict_groups AS
SELECT DISTINCT group_id as gid, local_id oid from tmp_dedup_events
WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''
UNION
SELECT DISTINCT group_id as gid, oa_original_id oid from tmp_dedup_events
WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != '';
INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by) SELECT DISTINCT
c1.oid, c2.oid, c1.gid, 'dedupWf', 'dedupWf'
FROM
tmp_conflict_groups c1
JOIN tmp_conflict_groups c2 ON (c1.gid = c2.gid)
WHERE
c1.oid != c2.oid
ON CONFLICT DO NOTHING;
COMMIT;

6
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html
View File

@ -1,8 +1,6 @@
<h2>Conflicts</h2>
<h4 class="text-muted" ng-if="conflicts.length == 0">No suggestions</h4>
<div class="input-group input-group-sm mb-3" ng-show="conflicts.length > 0">
<div class="input-group input-group-sm mb-3">
<input type="text" class="form-control" ng-model="conflictFilter" placeholder="Filter...">
<div class="input-group-append">
<span class="input-group-text text-outline-primary">Country:</span>
@ -19,6 +17,8 @@
</div>
</div>
<h5 class="text-muted" ng-if="conflicts.length == 0">No conflicts</h5>
<div class="card text-white mb-3" ng-repeat="w in conflicts | filter:conflictFilter" class="mb-2">
<div class="card-header bg-primary text-white py-1">Group {{$index+1}}</div>
<table class="table table-sm">

2
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html
View File

@ -14,7 +14,7 @@
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="valid organizations"># approved</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="to be approved by an administrator"># suggested</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="deleted by an administrator"># deleted</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="duplicates(not openorgs) of a valid organization"># duplicated</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="duplicates(not openorgs) of a valid organization"># duplicate</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="suggestions rejected by an administrator"># discarded</th>
<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="organizations hidden by the system (for example fixing a conflict)"># hidden</th>
</tr>

5
apps/dnet-orgs-database-application/src/test/java/SimpleTest.java Normal file
View File

@ -0,0 +1,5 @@
public class SimpleTest {
}