conflicts

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java

@@ -165,8 +165,6 @@ public class OrganizationController {
 	@GetMapping("/conflicts/byCountry/{country}")
 	public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
 
-		// databaseUtils.verifyConflictGroups(false);
-
 		if (UserInfo.isSuperAdmin(authentication)) {
 			return groupConflicts(conflictGroupViewRepository.findByCountry1OrCountry2(country, country).stream());
 		} else if (UserInfo.isSimpleUser(authentication) || UserInfo.isNationalAdmin(authentication)) {

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java

@@ -61,4 +61,9 @@ public class OpenaireDuplicate implements Serializable {
 		this.relType = relType;
 	}
 
+	@Override
+	public String toString() {
+		return String.format("OpenaireDuplicate [localId=%s, oaOriginalId=%s, relType=%s]", localId, oaOriginalId, relType);
+	}
+
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java

@@ -22,9 +22,13 @@ public interface OpenaireConflictRepository extends JpaRepository<OpenaireConfli
 	void resetGroupIds();
 
 	@Modifying
-	@Query(value = "update oa_conflicts set reltype = ?3, modified_by = ?4, modification_date = ?5 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
+	@Query(value = "update oa_conflicts set idgroup = null, reltype = ?3, modified_by = ?4, modification_date = ?5 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
-	void updateStatus(String id1, String id2, String status, String user, OffsetDateTime now);
+	void updateStatusAndResetGroup(String id1, String id2, String status, String user, OffsetDateTime now);
 
 	long countByGroupNull();
 
+	@Modifying
+	@Query(value = "update oa_conflicts set idgroup = null, reltype = ?2, modified_by = ?3, modification_date = ?4 where id1 = ?1 or id2 = ?1", nativeQuery = true)
+	void updateMultipleStatusAndResetGroup(String id, String type, String user, OffsetDateTime now);
+
 }

apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java

@@ -28,7 +28,6 @@ import org.springframework.web.bind.annotation.RequestBody;
 
 import eu.dnetlib.organizations.controller.UserRole;
 import eu.dnetlib.organizations.model.Acronym;
-import eu.dnetlib.organizations.model.OpenaireConflict;
 import eu.dnetlib.organizations.model.OpenaireConflictPK;
 import eu.dnetlib.organizations.model.OpenaireDuplicate;
 import eu.dnetlib.organizations.model.Organization;
@@ -243,47 +242,6 @@ public class DatabaseUtils {
 		}
 	}
 
-	/*
-	 * @Transactional public void verifyConflictGroups(final boolean forceUpdate) {
-	 *
-	 * if (forceUpdate || openaireConflictRepository.countByGroupNull() > 0) {
-	 *
-	 * log.info("Recreating conflicts group...");
-	 *
-	 * openaireConflictRepository.resetGroupIds();
-	 *
-	 * final Map<String, Set<String>> groups = new HashMap<>(); for (final OpenaireConflict w : openaireConflictRepository.findAll()) {
-	 * final List<String> list = findExistingGroupsForRel(w, groups); if (list.isEmpty()) { final String idGroup = generateGroupId();
-	 * groups.put(idGroup, new HashSet<>()); addToGroup(groups, idGroup, w); } else if (list.size() == 1) { addToGroup(groups, list.get(0),
-	 * w); } else { final String idGroup = generateGroupId(); groups.put(idGroup, new TreeSet<>()); list.forEach(id ->
-	 * groups.get(idGroup).addAll(groups.get(id))); list.forEach(id -> groups.remove(id)); addToGroup(groups, idGroup, w); } }
-	 *
-	 * for (final Entry<String, Set<String>> e : groups.entrySet()) { final String gid = e.getKey(); for (final String orgId : e.getValue())
-	 * { for (final OpenaireConflict oc : openaireConflictRepository.findById1AndGroupIsNull(orgId)) { oc.setGroup(gid);
-	 * openaireConflictRepository.save(oc); } for (final OpenaireConflict oc : openaireConflictRepository.findById2AndGroupIsNull(orgId)) {
-	 * oc.setGroup(gid); openaireConflictRepository.save(oc); } } }
-	 *
-	 * log.info("...conflict groups recreated"); } }
-	 *
-	 * private String generateGroupId() { return "group::" + UUID.randomUUID(); }
-	 */
-
-	private List<String> findExistingGroupsForRel(final OpenaireConflict w, final Map<String, Set<String>> groups) {
-		return groups.entrySet()
-			.stream()
-			.filter(e -> {
-				return e.getValue().contains(w.getId1()) || e.getValue().contains(w.getId2());
-			})
-			.map(e -> e.getKey())
-			.distinct()
-			.collect(Collectors.toList());
-	}
-
-	private void addToGroup(final Map<String, Set<String>> groups, final String gid, final OpenaireConflict w) {
-		groups.get(gid).add(w.getId1());
-		groups.get(gid).add(w.getId2());
-	}
-
 	@Transactional
 	public List<Relationship> makeRelation(final String id1, final String id2, final RelationType type) {
 		final Relationship r1 = new Relationship(id1, id2, type.toString());
@@ -385,10 +343,10 @@ public class DatabaseUtils {
 	}
 
 	@Transactional
-	public String fixConflict(final List<String> ids, final String user) {
+	public String fixConflict(final List<String> similarIds, final String user) {
 
 		final List<OrganizationView> views =
-			ids.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
+			similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
 
 		// I create a new org
 		final OrganizationView newOrg = new OrganizationView();
@@ -417,22 +375,26 @@ public class DatabaseUtils {
 		final String masterId = insertOrUpdateOrganization(newOrg, user, false);
 
 		// I hide the merged organizations
-		ids.forEach(id -> hideConflictOrgs(masterId, id));
+		similarIds.forEach(id -> hideConflictOrgs(masterId, id));
 
-		// I reassign the duplicated to the new org
+		// I reassign the duplicates to the new org
-		final List<OpenaireDuplicate> newDuplicates = ids.stream()
+		final List<OpenaireDuplicate> newDuplicates = similarIds.stream()
 			.map(openaireDuplicateRepository::findByLocalId)
 			.flatMap(l -> l.stream())
-			.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
 			.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType()))
 			.collect(Collectors.toList());
-		newDuplicates.forEach(d -> d.setLocalId(masterId));
+
+		openaireDuplicateRepository.saveAll(newDuplicates);
 
 		final OffsetDateTime now = OffsetDateTime.now();
 
-		for (int i = 0; i < ids.size(); i++) {
+		for (final String similarId : similarIds) {
-			for (int j = i + 1; j < ids.size(); j++) {
+			openaireConflictRepository.updateMultipleStatusAndResetGroup(similarId, SimilarityType.is_different.toString(), user, now);
-				openaireConflictRepository.updateStatus(ids.get(i), ids.get(j), SimilarityType.is_similar.toString(), user, now);
+		}
+
+		for (int i = 0; i < similarIds.size(); i++) {
+			for (int j = i + 1; j < similarIds.size(); j++) {
+				openaireConflictRepository.updateStatusAndResetGroup(similarIds.get(i), similarIds.get(j), SimilarityType.is_similar.toString(), user, now);
 			}
 		}
 

apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql

@@ -11,13 +11,13 @@ UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_c
 INSERT INTO organizations(id, name, country, status, created_by, modified_by) 
 SELECT oa_original_id, oa_name, oa_country, 'suggested', 'dedupWf', 'dedupWf' 
 FROM tmp_dedup_events 
-WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id = local_id
+WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND (oa_original_id = local_id OR local_id = '' OR local_id IS NULL)
 ON CONFLICT DO NOTHING;
 
 INSERT INTO organizations(id, name, country, status, created_by, modified_by) 
 SELECT oa_original_id, oa_name, oa_country, 'duplicate', 'dedupWf', 'dedupWf' 
 FROM tmp_dedup_events 
-WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id != local_id
+WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id is NOT NULL AND local_id != '' AND local_id != oa_original_id
 ON CONFLICT DO NOTHING;
 
 
@@ -32,12 +32,25 @@ WHERE local_id IS NOT NULL AND local_id != '' AND oa_original_id NOT LIKE 'openo
 ON CONFLICT DO NOTHING;
 
 
--- CONFLICTS
+-- CONFLICTS (I generate all the couples)
-INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by)
+CREATE TEMPORARY TABLE tmp_conflict_groups AS 
-SELECT local_id, oa_original_id, group_id, 'dedupWf', 'dedupWf'
+	SELECT DISTINCT group_id as gid, local_id oid from tmp_dedup_events
-FROM tmp_dedup_events
+	WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''
-WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != ''
+UNION 
+	SELECT DISTINCT group_id as gid, oa_original_id oid from tmp_dedup_events
+	WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id AND group_id IS NOT NULL AND group_id != '';
+
+INSERT INTO oa_conflicts (id1, id2, idgroup, created_by, modified_by) SELECT DISTINCT 
+	c1.oid, c2.oid, c1.gid, 'dedupWf', 'dedupWf'
+FROM
+	tmp_conflict_groups c1 
+	JOIN tmp_conflict_groups c2 ON (c1.gid = c2.gid) 
+WHERE 
+	c1.oid != c2.oid
 ON CONFLICT DO NOTHING;
 
 
+
+
+
 COMMIT;

apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/advanced/conflicts.html

@@ -1,8 +1,6 @@
 <h2>Conflicts</h2>
 
-<h4 class="text-muted" ng-if="conflicts.length == 0">No suggestions</h4>
+<div class="input-group input-group-sm mb-3">
-
-<div class="input-group input-group-sm mb-3" ng-show="conflicts.length > 0">
     <input type="text" class="form-control" ng-model="conflictFilter" placeholder="Filter...">
     <div class="input-group-append">
     	<span class="input-group-text text-outline-primary">Country:</span>
@@ -19,6 +17,8 @@
     </div>
 </div>
 
+<h5 class="text-muted" ng-if="conflicts.length == 0">No conflicts</h5>
+
 <div class="card text-white mb-3" ng-repeat="w in conflicts | filter:conflictFilter" class="mb-2">
 	<div class="card-header bg-primary text-white py-1">Group {{$index+1}}</div>
 	<table class="table table-sm">

apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/browse.html

@@ -14,7 +14,7 @@
 			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="valid organizations"># approved</th>
 			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="to be approved by an administrator"># suggested</th>
 			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="deleted by an administrator"># deleted</th>
-			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="duplicates(not openorgs) of a valid organization"># duplicated</th>
+			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="duplicates(not openorgs) of a valid organization"># duplicate</th>
 			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="suggestions rejected by an administrator"># discarded</th>
 			<th class="text-right text-nowrap col-1" ng-if="mode == 1" title="organizations hidden by the system (for example fixing a conflict)"># hidden</th>
 		</tr>

apps/dnet-orgs-database-application/src/test/java/SimpleTest.java

@@ -0,0 +1,5 @@
+
+
+public class SimpleTest {
+
+}