2020-07-03 12:09:22 +02:00
package eu.dnetlib.organizations.utils ;
import java.time.OffsetDateTime ;
import java.util.Arrays ;
import java.util.HashMap ;
import java.util.List ;
import java.util.Map ;
import java.util.Set ;
import java.util.stream.Collectors ;
import javax.transaction.Transactional ;
2020-10-14 16:28:00 +02:00
import org.apache.commons.io.IOUtils ;
2020-10-05 12:16:49 +02:00
import org.apache.commons.lang3.StringUtils ;
2020-09-28 16:53:20 +02:00
import org.apache.commons.logging.Log ;
import org.apache.commons.logging.LogFactory ;
2020-07-03 12:09:22 +02:00
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.cache.annotation.Cacheable ;
2020-07-29 11:31:18 +02:00
import org.springframework.jdbc.core.BeanPropertyRowMapper ;
2020-07-03 12:09:22 +02:00
import org.springframework.jdbc.core.JdbcTemplate ;
import org.springframework.stereotype.Component ;
import org.springframework.web.bind.annotation.RequestBody ;
import eu.dnetlib.organizations.controller.UserRole ;
import eu.dnetlib.organizations.model.Acronym ;
import eu.dnetlib.organizations.model.OpenaireConflict ;
import eu.dnetlib.organizations.model.OpenaireConflictPK ;
2020-09-29 15:31:56 +02:00
import eu.dnetlib.organizations.model.OpenaireDuplicate ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.model.Organization ;
import eu.dnetlib.organizations.model.OtherIdentifier ;
import eu.dnetlib.organizations.model.OtherName ;
import eu.dnetlib.organizations.model.Relationship ;
import eu.dnetlib.organizations.model.Url ;
import eu.dnetlib.organizations.model.User ;
import eu.dnetlib.organizations.model.UserCountry ;
2020-07-29 11:31:18 +02:00
import eu.dnetlib.organizations.model.utils.BrowseEntry ;
2020-09-28 16:53:20 +02:00
import eu.dnetlib.organizations.model.utils.OrganizationConflict ;
2020-10-13 14:48:04 +02:00
import eu.dnetlib.organizations.model.utils.TempBrowseEntry ;
2020-10-08 15:10:11 +02:00
import eu.dnetlib.organizations.model.utils.VocabularyTerm ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.model.view.OrganizationView ;
import eu.dnetlib.organizations.model.view.UserView ;
import eu.dnetlib.organizations.repository.AcronymRepository ;
import eu.dnetlib.organizations.repository.OpenaireConflictRepository ;
2020-09-29 15:31:56 +02:00
import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository ;
2020-07-03 12:09:22 +02:00
import eu.dnetlib.organizations.repository.OrganizationRepository ;
import eu.dnetlib.organizations.repository.OtherIdentifierRepository ;
import eu.dnetlib.organizations.repository.OtherNameRepository ;
import eu.dnetlib.organizations.repository.RelationshipRepository ;
import eu.dnetlib.organizations.repository.UrlRepository ;
import eu.dnetlib.organizations.repository.UserCountryRepository ;
import eu.dnetlib.organizations.repository.UserRepository ;
@Component
public class DatabaseUtils {
@Autowired
private AcronymRepository acronymRepository ;
@Autowired
private OrganizationRepository organizationRepository ;
@Autowired
private OtherIdentifierRepository otherIdentifierRepository ;
@Autowired
private OtherNameRepository otherNameRepository ;
@Autowired
private UrlRepository urlRepository ;
@Autowired
private RelationshipRepository relationshipRepository ;
@Autowired
private UserRepository userRepository ;
@Autowired
private UserCountryRepository userCountryRepository ;
@Autowired
private OpenaireConflictRepository openaireConflictRepository ;
@Autowired
2020-09-29 15:31:56 +02:00
private OpenaireDuplicateRepository openaireDuplicateRepository ;
@Autowired
2020-07-03 12:09:22 +02:00
private JdbcTemplate jdbcTemplate ;
2020-09-28 16:53:20 +02:00
private static final Log log = LogFactory . getLog ( DatabaseUtils . class ) ;
2020-07-03 12:09:22 +02:00
public enum VocabularyTable {
2020-07-29 11:31:18 +02:00
languages ,
countries ,
org_types ,
id_types ,
rel_types ,
simrel_types
2020-07-03 12:09:22 +02:00
}
@Transactional
2020-10-05 12:16:49 +02:00
public String insertOrUpdateOrganization ( final OrganizationView orgView , final String user ) {
2020-10-07 17:04:29 +02:00
final boolean alreadyApproved = StringUtils . equals ( orgView . getStatus ( ) , OrganizationStatus . approved . toString ( ) ) ;
2020-10-05 12:16:49 +02:00
2020-10-07 17:04:29 +02:00
final String oldId = orgView . getId ( ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
if ( StringUtils . isBlank ( orgView . getId ( ) ) ) {
2020-10-05 12:16:49 +02:00
orgView . setId ( null ) ;
2020-10-07 17:04:29 +02:00
} else if ( ! alreadyApproved ) {
cleanOldRelations ( oldId ) ;
organizationRepository . deleteById ( oldId ) ;
orgView . setId ( null ) ;
} else {
cleanOldRelations ( orgView . getId ( ) ) ;
2020-10-05 12:16:49 +02:00
}
2020-10-07 17:04:29 +02:00
final Organization org = new Organization ( orgView . getId ( ) ,
2020-07-29 11:31:18 +02:00
orgView . getName ( ) ,
orgView . getType ( ) ,
orgView . getLat ( ) , orgView . getLng ( ) ,
2020-10-06 15:06:12 +02:00
orgView . getCity ( ) , orgView . getCountry ( ) ,
OrganizationStatus . approved . toString ( ) ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
final String newId = organizationRepository . save ( org ) . getId ( ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
makeNewRelations ( newId , orgView ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
updateHistoryFields ( newId , user , alreadyApproved ) ;
2020-07-03 12:09:22 +02:00
2020-10-07 17:04:29 +02:00
return newId ;
2020-07-03 12:09:22 +02:00
}
private void updateHistoryFields ( final String id , final String user , final boolean update ) {
2020-10-07 17:04:29 +02:00
2020-07-03 12:09:22 +02:00
final OffsetDateTime now = OffsetDateTime . now ( ) ;
if ( update ) {
organizationRepository . updateModificationDate ( id , user , now ) ;
} else {
organizationRepository . updateCreationDate ( id , user , now ) ;
organizationRepository . updateModificationDate ( id , user , now ) ;
}
}
2020-09-29 15:31:56 +02:00
@Transactional
2020-10-09 16:00:12 +02:00
public void saveDuplicates ( final List < OpenaireDuplicate > simrels , final String email ) {
2020-09-29 15:31:56 +02:00
final OffsetDateTime now = OffsetDateTime . now ( ) ;
final List < OpenaireDuplicate > list = openaireDuplicateRepository . saveAll ( simrels ) ;
2020-10-09 16:00:12 +02:00
2020-09-29 15:31:56 +02:00
list . forEach ( d - > openaireDuplicateRepository . updateModificationDate ( d . getLocalId ( ) , d . getOaOriginalId ( ) , email , now ) ) ;
2020-10-09 16:00:12 +02:00
2020-09-29 15:31:56 +02:00
}
2020-10-07 17:04:29 +02:00
private void makeNewRelations ( final String orgId , final OrganizationView orgView ) {
2020-07-03 12:09:22 +02:00
orgView . getAcronyms ( ) . forEach ( s - > acronymRepository . save ( new Acronym ( orgId , s ) ) ) ;
orgView . getOtherNames ( ) . forEach ( n - > otherNameRepository . save ( new OtherName ( orgId , n . getName ( ) , n . getLang ( ) ) ) ) ;
orgView . getOtherIdentifiers ( ) . forEach ( id - > otherIdentifierRepository . save ( new OtherIdentifier ( orgId , id . getId ( ) , id . getType ( ) ) ) ) ;
orgView . getUrls ( ) . forEach ( u - > urlRepository . save ( new Url ( orgId , u ) ) ) ;
orgView . getRelations ( ) . forEach ( r - > makeRelation ( orgId , r . getRelatedOrgId ( ) , RelationType . valueOf ( r . getType ( ) ) ) ) ;
}
private void cleanOldRelations ( final String id ) {
acronymRepository . deleteByOrgId ( id ) ;
otherNameRepository . deleteByOrgId ( id ) ;
otherIdentifierRepository . deleteByOrgId ( id ) ;
urlRepository . deleteByOrgId ( id ) ;
relationshipRepository . deleteById1 ( id ) ;
relationshipRepository . deleteById2 ( id ) ;
}
@Cacheable ( " vocs " )
2020-10-08 15:10:11 +02:00
public List < VocabularyTerm > listValuesOfVocabularyTable ( final VocabularyTable table ) {
final String sql = " select val as value, name as name from " + table ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( VocabularyTerm . class ) ) ;
2020-07-03 12:09:22 +02:00
}
@Cacheable ( " countries_for_user " )
2020-10-08 15:10:11 +02:00
public List < VocabularyTerm > listCountriesForUser ( final String name ) {
final String sql =
" select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? " ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( VocabularyTerm . class ) , name ) ;
2020-07-03 12:09:22 +02:00
}
@Transactional
public void saveUser ( @RequestBody final UserView userView ) {
final User user = userRepository . findById ( userView . getEmail ( ) ) . orElseThrow ( ( ) - > new RuntimeException ( " User not found " ) ) ;
user . setRole ( userView . getRole ( ) ) ;
user . setValid ( userView . isValid ( ) ) ;
userRepository . save ( user ) ;
userCountryRepository . deleteByEmail ( userView . getEmail ( ) ) ;
if ( userView . getCountries ( ) ! = null ) {
userCountryRepository
2020-07-29 11:31:18 +02:00
. saveAll ( Arrays . stream ( userView . getCountries ( ) ) . map ( c - > new UserCountry ( userView . getEmail ( ) , c ) ) . collect ( Collectors . toList ( ) ) ) ;
2020-07-03 12:09:22 +02:00
}
}
@Transactional
public void deleteUser ( final String email ) {
userCountryRepository . deleteByEmail ( email ) ;
userRepository . deleteById ( email ) ;
}
@Transactional
public void newUser ( final String email , final List < String > countries ) {
final User user = new User ( ) ;
user . setEmail ( email ) ;
user . setRole ( UserRole . PENDING . name ( ) ) ;
user . setValid ( false ) ;
userRepository . save ( user ) ;
if ( countries ! = null ) {
userCountryRepository . saveAll ( countries . stream ( ) . map ( c - > new UserCountry ( email , c ) ) . collect ( Collectors . toList ( ) ) ) ;
}
}
2020-10-14 16:28:00 +02:00
/ *
* @Transactional public void verifyConflictGroups ( final boolean forceUpdate ) {
*
* if ( forceUpdate | | openaireConflictRepository . countByGroupNull ( ) > 0 ) {
*
* log . info ( " Recreating conflicts group... " ) ;
*
* openaireConflictRepository . resetGroupIds ( ) ;
*
* final Map < String , Set < String > > groups = new HashMap < > ( ) ; for ( final OpenaireConflict w : openaireConflictRepository . findAll ( ) ) {
* final List < String > list = findExistingGroupsForRel ( w , groups ) ; if ( list . isEmpty ( ) ) { final String idGroup = generateGroupId ( ) ;
* groups . put ( idGroup , new HashSet < > ( ) ) ; addToGroup ( groups , idGroup , w ) ; } else if ( list . size ( ) = = 1 ) { addToGroup ( groups , list . get ( 0 ) ,
* w ) ; } else { final String idGroup = generateGroupId ( ) ; groups . put ( idGroup , new TreeSet < > ( ) ) ; list . forEach ( id - >
* groups . get ( idGroup ) . addAll ( groups . get ( id ) ) ) ; list . forEach ( id - > groups . remove ( id ) ) ; addToGroup ( groups , idGroup , w ) ; } }
*
* for ( final Entry < String , Set < String > > e : groups . entrySet ( ) ) { final String gid = e . getKey ( ) ; for ( final String orgId : e . getValue ( ) )
* { for ( final OpenaireConflict oc : openaireConflictRepository . findById1AndGroupIsNull ( orgId ) ) { oc . setGroup ( gid ) ;
* openaireConflictRepository . save ( oc ) ; } for ( final OpenaireConflict oc : openaireConflictRepository . findById2AndGroupIsNull ( orgId ) ) {
* oc . setGroup ( gid ) ; openaireConflictRepository . save ( oc ) ; } } }
*
* log . info ( " ...conflict groups recreated " ) ; } }
*
* private String generateGroupId ( ) { return " group:: " + UUID . randomUUID ( ) ; }
* /
2020-07-03 12:09:22 +02:00
private List < String > findExistingGroupsForRel ( final OpenaireConflict w , final Map < String , Set < String > > groups ) {
return groups . entrySet ( )
2020-07-29 11:31:18 +02:00
. stream ( )
. filter ( e - > {
return e . getValue ( ) . contains ( w . getId1 ( ) ) | | e . getValue ( ) . contains ( w . getId2 ( ) ) ;
} )
. map ( e - > e . getKey ( ) )
. distinct ( )
. collect ( Collectors . toList ( ) ) ;
2020-07-03 12:09:22 +02:00
}
private void addToGroup ( final Map < String , Set < String > > groups , final String gid , final OpenaireConflict w ) {
groups . get ( gid ) . add ( w . getId1 ( ) ) ;
groups . get ( gid ) . add ( w . getId2 ( ) ) ;
}
@Transactional
public List < Relationship > makeRelation ( final String id1 , final String id2 , final RelationType type ) {
final Relationship r1 = new Relationship ( id1 , id2 , type . toString ( ) ) ;
final Relationship r2 = new Relationship ( id2 , id1 , type . getInverse ( ) . toString ( ) ) ;
relationshipRepository . save ( r1 ) ;
relationshipRepository . save ( r2 ) ;
return Arrays . asList ( r1 , r2 ) ;
}
2020-07-29 11:31:18 +02:00
// BROWSE BY COUNTRY
public List < BrowseEntry > browseCountries ( ) {
2020-10-01 16:30:32 +02:00
final String sql =
2020-10-13 14:48:04 +02:00
" select o.country as code, c.name as name, o.status as group, count(o.status) as count from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name, o.status " ;
return listBrowseEntries ( sql ) ;
2020-07-29 11:31:18 +02:00
}
// BROWSE BY COUNTRY FOR USER
public List < BrowseEntry > browseCountriesForUser ( final String email ) {
final String sql =
2020-10-13 14:48:04 +02:00
" select o.country as code, c.name as name, o.status as group, count(o.status) as count from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name, o.status " ;
return listBrowseEntries ( sql , email ) ;
2020-07-29 11:31:18 +02:00
}
// BROWSE BY ORG TYPE
public List < BrowseEntry > browseTypes ( ) {
2020-10-01 16:30:32 +02:00
final String sql =
2020-10-13 14:48:04 +02:00
" select type as code, type as name, status as group, count(status) as count from organizations group by type, status " ;
return listBrowseEntries ( sql ) ;
2020-07-29 11:31:18 +02:00
}
// BROWSE BY ORG TYPE FOR USER
public List < BrowseEntry > browseTypesForUser ( final String email ) {
2020-10-13 14:48:04 +02:00
final String sql = " select o.type as code, o.type as name, "
+ " o.status as group, count(o.status) as count "
2020-07-29 11:31:18 +02:00
+ " from organizations o "
+ " left outer join user_countries uc on (uc.country = o.country) "
+ " where uc.email=? "
2020-10-13 14:48:04 +02:00
+ " group by o.type, o.status " ;
return listBrowseEntries ( sql , email ) ;
}
private List < BrowseEntry > listBrowseEntries ( final String sql , final Object . . . params ) {
final Map < String , BrowseEntry > map = new HashMap < > ( ) ;
for ( final TempBrowseEntry t : jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( TempBrowseEntry . class ) , params ) ) {
if ( StringUtils . isNotBlank ( t . getCode ( ) ) ) {
if ( ! map . containsKey ( t . getCode ( ) ) ) {
final BrowseEntry e = new BrowseEntry ( ) ;
e . setCode ( t . getCode ( ) ) ;
e . setName ( t . getName ( ) ) ;
map . put ( t . getCode ( ) , e ) ;
}
map . get ( t . getCode ( ) ) . getValues ( ) . put ( t . getGroup ( ) , t . getCount ( ) ) ;
}
}
2020-07-29 11:31:18 +02:00
2020-10-13 14:48:04 +02:00
return map . values ( ) . stream ( ) . sorted ( ( o1 , o2 ) - > StringUtils . compare ( o1 . getName ( ) , o2 . getName ( ) ) ) . collect ( Collectors . toList ( ) ) ;
2020-07-29 11:31:18 +02:00
}
2020-09-28 16:53:20 +02:00
public List < OrganizationConflict > listConflictsForId ( final String id ) {
final String sql =
" select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? " ;
return jdbcTemplate . query ( sql , new BeanPropertyRowMapper < > ( OrganizationConflict . class ) , id ) ;
}
2020-09-29 11:48:45 +02:00
@Transactional
2020-10-09 16:00:12 +02:00
public void importDedupEvents ( ) {
2020-09-28 16:53:20 +02:00
try {
2020-10-14 16:28:00 +02:00
log . info ( " Importing conflicts and duplicates... " ) ;
jdbcTemplate . update ( IOUtils . toString ( getClass ( ) . getResourceAsStream ( " /sql/importDedupEvents.sql " ) ) ) ;
log . info ( " ...done " ) ;
2020-10-09 16:00:12 +02:00
// verifyConflictGroups(true);
2020-09-28 16:53:20 +02:00
} catch ( final Exception e ) {
2020-09-29 11:34:31 +02:00
log . error ( " Error importing conflicts and duplicates " , e ) ;
2020-09-28 16:53:20 +02:00
}
}
2020-10-07 17:04:29 +02:00
@Transactional
public List < Relationship > fixDuplicate ( final String masterId , final String otherId ) {
organizationRepository . updateStatus ( otherId , OrganizationStatus . hidden . toString ( ) ) ;
openaireConflictRepository . findById ( new OpenaireConflictPK ( masterId , otherId ) ) . ifPresent ( openaireConflictRepository : : delete ) ;
openaireConflictRepository . findById ( new OpenaireConflictPK ( otherId , masterId ) ) . ifPresent ( openaireConflictRepository : : delete ) ;
// TODO Merge the organizations ???
return makeRelation ( masterId , otherId , RelationType . Merges ) ;
}
2020-07-03 12:09:22 +02:00
}