2020-07-03 12:09:22 +02:00
package eu.dnetlib.organizations.utils ;
import java.time.OffsetDateTime ;
import java.util.Arrays ;
import java.util.HashMap ;
import java.util.HashSet ;
import java.util.List ;
import java.util.Map ;
import java.util.Map.Entry ;
import java.util.Set ;
import java.util.TreeSet ;
import java.util.UUID ;
import java.util.stream.Collectors ;
import javax.transaction.Transactional ;
2020-10-05 12:16:49 +02:00
import org.apache.commons.lang3.StringUtils ;
2020-09-28 16:53:20 +02:00
import org.apache.commons.logging.Log ;
import org.apache.commons.logging.LogFactory ;
2020-07-03 12:09:22 +02:00
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.cache.annotation.Cacheable ;
2020-07-29 11:31:18 +02:00
import org.springframework.jdbc.core.BeanPropertyRowMapper ;
2020-07-03 12:09:22 +02:00
import org.springframework.jdbc.core.JdbcTemplate ;
import org.springframework.stereotype.Component ;
import org.springframework.web.bind.annotation.RequestBody ;
import eu.dnetlib.organizations.controller.UserRole ;
import eu.dnetlib.organizations.model.Acronym ;
import eu.dnetlib.organizations.model.OpenaireConflict ;
import eu.dnetlib.organizations.model.OpenaireConflictPK ;
2020-09-29 15:31:56 +02:00
import eu.dnetlib.organizations.model.OpenaireDuplicate ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.model.Organization ;
import eu.dnetlib.organizations.model.OtherIdentifier ;
import eu.dnetlib.organizations.model.OtherName ;
import eu.dnetlib.organizations.model.Relationship ;
import eu.dnetlib.organizations.model.Url ;
import eu.dnetlib.organizations.model.User ;
import eu.dnetlib.organizations.model.UserCountry ;
2020-07-29 11:31:18 +02:00
import eu.dnetlib.organizations.model.utils.BrowseEntry ;
2020-09-28 16:53:20 +02:00
import eu.dnetlib.organizations.model.utils.OrganizationConflict ;
2020-10-08 15:10:11 +02:00
import eu.dnetlib.organizations.model.utils.VocabularyTerm ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.model.view.OrganizationView ;
import eu.dnetlib.organizations.model.view.UserView ;
import eu.dnetlib.organizations.repository.AcronymRepository ;
import eu.dnetlib.organizations.repository.OpenaireConflictRepository ;
2020-09-29 15:31:56 +02:00
import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.repository.OrganizationRepository ;
import eu.dnetlib.organizations.repository.OtherIdentifierRepository ;
import eu.dnetlib.organizations.repository.OtherNameRepository ;
import eu.dnetlib.organizations.repository.RelationshipRepository ;
import eu.dnetlib.organizations.repository.UrlRepository ;
import eu.dnetlib.organizations.repository.UserCountryRepository ;
import eu.dnetlib.organizations.repository.UserRepository ;
@Component
public class DatabaseUtils {
@Autowired
private AcronymRepository acronymRepository ;
@Autowired
private OrganizationRepository organizationRepository ;
@Autowired
private OtherIdentifierRepository otherIdentifierRepository ;
@Autowired
private OtherNameRepository otherNameRepository ;
@Autowired
private UrlRepository urlRepository ;
@Autowired
private RelationshipRepository relationshipRepository ;
@Autowired
private UserRepository userRepository ;
@Autowired
private UserCountryRepository userCountryRepository ;
@Autowired
private OpenaireConflictRepository openaireConflictRepository ;
@Autowired
2020-09-29 15:31:56 +02:00
private OpenaireDuplicateRepository openaireDuplicateRepository ;
@Autowired
2020-07-03 12:09:22 +02:00
private JdbcTemplate jdbcTemplate ;
2020-09-28 16:53:20 +02:00
private static final Log log = LogFactory . getLog ( DatabaseUtils . class ) ;
2020-07-03 12:09:22 +02:00
public enum VocabularyTable {
2020-07-29 11:31:18 +02:00
languages ,
countries ,
org_types ,
id_types ,
rel_types ,
simrel_types
2020-07-03 12:09:22 +02:00
}
@Transactional
2020-10-05 12:16:49 +02:00
public String insertOrUpdateOrganization ( final OrganizationView orgView , final String user ) {
2020-10-07 17:04:29 +02:00
final boolean alreadyApproved = StringUtils . equals ( orgView . getStatus ( ) , OrganizationStatus . approved . toString ( ) ) ;
2020-10-05 12:16:49 +02:00
2020-10-07 17:04:29 +02:00
final String oldId = orgView . getId ( ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
if ( StringUtils . isBlank ( orgView . getId ( ) ) ) {
2020-10-05 12:16:49 +02:00
orgView . setId ( null ) ;
2020-10-07 17:04:29 +02:00
} else if ( ! alreadyApproved ) {
cleanOldRelations ( oldId ) ;
organizationRepository . deleteById ( oldId ) ;
orgView . setId ( null ) ;
} else {
cleanOldRelations ( orgView . getId ( ) ) ;
2020-10-05 12:16:49 +02:00
}
2020-10-07 17:04:29 +02:00
final Organization org = new Organization ( orgView . getId ( ) ,
2020-07-29 11:31:18 +02:00
orgView . getName ( ) ,
orgView . getType ( ) ,
orgView . getLat ( ) , orgView . getLng ( ) ,
2020-10-06 15:06:12 +02:00
orgView . getCity ( ) , orgView . getCountry ( ) ,
OrganizationStatus . approved . toString ( ) ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
final String newId = organizationRepository . save ( org ) . getId ( ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
makeNewRelations ( newId , orgView ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
updateHistoryFields ( newId , user , alreadyApproved ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
return newId ;
2020-07-03 12:09:22 +02:00
}
private void updateHistoryFields ( final String id , final String user , final boolean update ) {
2020-10-07 17:04:29 +02:00
2020-07-03 12:09:22 +02:00
final OffsetDateTime now = OffsetDateTime . now ( ) ;
if ( update ) {
organizationRepository . updateModificationDate ( id , user , now ) ;
} else {
organizationRepository . updateCreationDate ( id , user , now ) ;
organizationRepository . updateModificationDate ( id , user , now ) ;
}
}
2020-09-29 15:31:56 +02:00
@Transactional
2020-10-09 16:00:12 +02:00
public void saveDuplicates ( final List < OpenaireDuplicate > simrels , final String email ) {
2020-09-29 15:31:56 +02:00
final OffsetDateTime now = OffsetDateTime . now ( ) ;
final List < OpenaireDuplicate > list = openaireDuplicateRepository . saveAll ( simrels ) ;
2020-10-09 16:00:12 +02:00
2020-09-29 15:31:56 +02:00
list . forEach ( d - > openaireDuplicateRepository . updateModificationDate ( d . getLocalId ( ) , d . getOaOriginalId ( ) , email , now ) ) ;
2020-10-09 16:00:12 +02:00
2020-09-29 15:31:56 +02:00
}
2020-10-07 17:04:29 +02:00
private void makeNewRelations ( final String orgId , final OrganizationView orgView ) {
2020-07-03 12:09:22 +02:00
orgView . getAcronyms ( ) . forEach ( s - > acronymRepository . save ( new Acronym ( orgId , s ) ) ) ;
orgView . getOtherNames ( ) . forEach ( n - > otherNameRepository . save ( new OtherName ( orgId , n . getName ( ) , n . getLang ( ) ) ) ) ;
orgView . getOtherIdentifiers ( ) . forEach ( id - > otherIdentifierRepository . save ( new OtherIdentifier ( orgId , id . getId ( ) , id . getType ( ) ) ) ) ;
orgView . getUrls ( ) . forEach ( u - > urlRepository . save ( new Url ( orgId , u ) ) ) ;
orgView . getRelations ( ) . forEach ( r - > makeRelation ( orgId , r . getRelatedOrgId ( ) , RelationType . valueOf ( r . getType ( ) ) ) ) ;
}
private void cleanOldRelations ( final String id ) {
acronymRepository . deleteByOrgId ( id ) ;
otherNameRepository . deleteByOrgId ( id ) ;
otherIdentifierRepository . deleteByOrgId ( id ) ;
urlRepository . deleteByOrgId ( id ) ;
relationshipRepository . deleteById1 ( id ) ;
relationshipRepository . deleteById2 ( id ) ;
}
@Cacheable ( " vocs " )
2020-10-08 15:10:11 +02:00
public List < VocabularyTerm > listValuesOfVocabularyTable ( final VocabularyTable table ) {
final String sql = " select val as value, name as name from " + table ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( VocabularyTerm . class ) ) ;
2020-07-03 12:09:22 +02:00
}
@Cacheable ( " countries_for_user " )
2020-10-08 15:10:11 +02:00
public List < VocabularyTerm > listCountriesForUser ( final String name ) {
final String sql =
" select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? " ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( VocabularyTerm . class ) , name ) ;
2020-07-03 12:09:22 +02:00
}
@Transactional
public void saveUser ( @RequestBody final UserView userView ) {
final User user = userRepository . findById ( userView . getEmail ( ) ) . orElseThrow ( ( ) - > new RuntimeException ( " User not found " ) ) ;
user . setRole ( userView . getRole ( ) ) ;
user . setValid ( userView . isValid ( ) ) ;
userRepository . save ( user ) ;
userCountryRepository . deleteByEmail ( userView . getEmail ( ) ) ;
if ( userView . getCountries ( ) ! = null ) {
userCountryRepository
2020-07-29 11:31:18 +02:00
. saveAll ( Arrays . stream ( userView . getCountries ( ) ) . map ( c - > new UserCountry ( userView . getEmail ( ) , c ) ) . collect ( Collectors . toList ( ) ) ) ;
2020-07-03 12:09:22 +02:00
}
}
@Transactional
public void deleteUser ( final String email ) {
userCountryRepository . deleteByEmail ( email ) ;
userRepository . deleteById ( email ) ;
}
@Transactional
public void newUser ( final String email , final List < String > countries ) {
final User user = new User ( ) ;
user . setEmail ( email ) ;
user . setRole ( UserRole . PENDING . name ( ) ) ;
user . setValid ( false ) ;
userRepository . save ( user ) ;
if ( countries ! = null ) {
userCountryRepository . saveAll ( countries . stream ( ) . map ( c - > new UserCountry ( email , c ) ) . collect ( Collectors . toList ( ) ) ) ;
}
}
@Transactional
public void verifyConflictGroups ( final boolean forceUpdate ) {
if ( forceUpdate | | openaireConflictRepository . countByGroupNull ( ) > 0 ) {
2020-09-29 11:48:45 +02:00
log . info ( " Recreating conflicts group... " ) ;
2020-07-03 12:09:22 +02:00
openaireConflictRepository . resetGroupIds ( ) ;
final Map < String , Set < String > > groups = new HashMap < > ( ) ;
for ( final OpenaireConflict w : openaireConflictRepository . findAll ( ) ) {
final List < String > list = findExistingGroupsForRel ( w , groups ) ;
if ( list . isEmpty ( ) ) {
final String idGroup = generateGroupId ( ) ;
groups . put ( idGroup , new HashSet < > ( ) ) ;
addToGroup ( groups , idGroup , w ) ;
} else if ( list . size ( ) = = 1 ) {
addToGroup ( groups , list . get ( 0 ) , w ) ;
} else {
final String idGroup = generateGroupId ( ) ;
groups . put ( idGroup , new TreeSet < > ( ) ) ;
list . forEach ( id - > groups . get ( idGroup ) . addAll ( groups . get ( id ) ) ) ;
list . forEach ( id - > groups . remove ( id ) ) ;
addToGroup ( groups , idGroup , w ) ;
}
}
for ( final Entry < String , Set < String > > e : groups . entrySet ( ) ) {
final String gid = e . getKey ( ) ;
for ( final String orgId : e . getValue ( ) ) {
for ( final OpenaireConflict oc : openaireConflictRepository . findById1AndGroupIsNull ( orgId ) ) {
oc . setGroup ( gid ) ;
openaireConflictRepository . save ( oc ) ;
}
for ( final OpenaireConflict oc : openaireConflictRepository . findById2AndGroupIsNull ( orgId ) ) {
oc . setGroup ( gid ) ;
openaireConflictRepository . save ( oc ) ;
}
}
}
2020-09-29 11:48:45 +02:00
log . info ( " ...conflicts group recreated " ) ;
2020-07-03 12:09:22 +02:00
}
}
private String generateGroupId ( ) {
return " group:: " + UUID . randomUUID ( ) ;
}
private List < String > findExistingGroupsForRel ( final OpenaireConflict w , final Map < String , Set < String > > groups ) {
return groups . entrySet ( )
2020-07-29 11:31:18 +02:00
. stream ( )
. filter ( e - > {
return e . getValue ( ) . contains ( w . getId1 ( ) ) | | e . getValue ( ) . contains ( w . getId2 ( ) ) ;
} )
. map ( e - > e . getKey ( ) )
. distinct ( )
. collect ( Collectors . toList ( ) ) ;
2020-07-03 12:09:22 +02:00
}
private void addToGroup ( final Map < String , Set < String > > groups , final String gid , final OpenaireConflict w ) {
groups . get ( gid ) . add ( w . getId1 ( ) ) ;
groups . get ( gid ) . add ( w . getId2 ( ) ) ;
}
@Transactional
public List < Relationship > makeRelation ( final String id1 , final String id2 , final RelationType type ) {
final Relationship r1 = new Relationship ( id1 , id2 , type . toString ( ) ) ;
final Relationship r2 = new Relationship ( id2 , id1 , type . getInverse ( ) . toString ( ) ) ;
relationshipRepository . save ( r1 ) ;
relationshipRepository . save ( r2 ) ;
return Arrays . asList ( r1 , r2 ) ;
}
2020-07-29 11:31:18 +02:00
// BROWSE BY COUNTRY
public List < BrowseEntry > browseCountries ( ) {
2020-10-01 16:30:32 +02:00
final String sql =
2020-10-08 15:10:11 +02:00
" select o.country as value, c.name as name, sum(case when status='approved' then 1 else 0 end) as approved, sum(case when status='pending' then 1 else 0 end) as pending from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name order by approved desc " ;
2020-07-29 11:31:18 +02:00
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( BrowseEntry . class ) ) ;
}
// BROWSE BY COUNTRY FOR USER
public List < BrowseEntry > browseCountriesForUser ( final String email ) {
final String sql =
2020-10-08 15:10:11 +02:00
" select o.country as value, c.name as name, sum(case when status='approved' then 1 else 0 end) as approved, sum(case when status='pending' then 1 else 0 end) as pending from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name order by approved desc " ;
2020-07-29 11:31:18 +02:00
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( BrowseEntry . class ) , email ) ;
}
// BROWSE BY ORG TYPE
public List < BrowseEntry > browseTypes ( ) {
2020-10-01 16:30:32 +02:00
final String sql =
2020-10-08 15:10:11 +02:00
" select type as value, type as name, sum(case when status='approved' then 1 else 0 end) as approved, sum(case when status='pending' then 1 else 0 end) as pending from organizations group by type order by approved desc " ;
2020-07-29 11:31:18 +02:00
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( BrowseEntry . class ) ) ;
}
// BROWSE BY ORG TYPE FOR USER
public List < BrowseEntry > browseTypesForUser ( final String email ) {
2020-10-08 15:10:11 +02:00
final String sql = " select o.type as value, o.type as name, "
2020-10-06 15:06:12 +02:00
+ " sum(case when status='approved' then 1 else 0 end) as approved, "
+ " sum(case when status='pending' then 1 else 0 end) as pending "
2020-07-29 11:31:18 +02:00
+ " from organizations o "
+ " left outer join user_countries uc on (uc.country = o.country) "
+ " where uc.email=? "
+ " group by o.type "
2020-10-01 16:30:32 +02:00
+ " order by approved desc; " ;
2020-07-29 11:31:18 +02:00
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( BrowseEntry . class ) , email ) ;
}
2020-09-28 16:53:20 +02:00
public List < OrganizationConflict > listConflictsForId ( final String id ) {
final String sql =
" select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? " ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( OrganizationConflict . class ) , id ) ;
}
2020-09-29 11:48:45 +02:00
@Transactional
2020-10-09 16:00:12 +02:00
public void importDedupEvents ( ) {
2020-09-28 16:53:20 +02:00
try {
2020-10-09 16:00:12 +02:00
// log.info("Importing conflicts and duplicates...");
// jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql")));
// log.info("...done");
// verifyConflictGroups(true);
2020-09-28 16:53:20 +02:00
} catch ( final Exception e ) {
2020-09-29 11:34:31 +02:00
log . error ( " Error importing conflicts and duplicates " , e ) ;
2020-09-28 16:53:20 +02:00
}
}
2020-10-07 17:04:29 +02:00
@Transactional
public List < Relationship > fixDuplicate ( final String masterId , final String otherId ) {
organizationRepository . updateStatus ( otherId , OrganizationStatus . hidden . toString ( ) ) ;
openaireConflictRepository . findById ( new OpenaireConflictPK ( masterId , otherId ) ) . ifPresent ( openaireConflictRepository : : delete ) ;
openaireConflictRepository . findById ( new OpenaireConflictPK ( otherId , masterId ) ) . ifPresent ( openaireConflictRepository : : delete ) ;
// TODO Merge the organizations ???
return makeRelation ( masterId , otherId , RelationType . Merges ) ;
}
2020-07-03 12:09:22 +02:00
}