2023-02-06 15:53:07 +01:00
package eu.dnetlib.data.mdstore ;
2023-02-06 14:50:10 +01:00
2023-02-27 11:43:28 +01:00
import java.time.LocalDateTime ;
import java.time.ZoneOffset ;
2023-02-08 12:23:53 +01:00
import java.util.Comparator ;
2023-02-10 10:27:09 +01:00
import java.util.LinkedHashMap ;
2023-02-06 14:50:10 +01:00
import java.util.List ;
2023-02-10 10:27:09 +01:00
import java.util.Map ;
import java.util.Set ;
import java.util.UUID ;
2023-02-08 12:23:53 +01:00
import java.util.function.Function ;
2023-02-06 14:50:10 +01:00
import java.util.stream.Collectors ;
2023-04-18 16:13:42 +02:00
import java.util.stream.Stream ;
2023-02-08 12:23:53 +01:00
import java.util.stream.StreamSupport ;
2023-02-06 14:50:10 +01:00
import javax.transaction.Transactional ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.jdbc.core.JdbcTemplate ;
2023-02-10 10:27:09 +01:00
import org.springframework.stereotype.Service ;
2023-02-06 14:50:10 +01:00
2023-02-10 14:54:37 +01:00
import eu.dnetlib.data.mdstore.backends.DefaultBackend ;
import eu.dnetlib.data.mdstore.backends.HdfsBackend ;
import eu.dnetlib.data.mdstore.backends.MDStoreBackend ;
import eu.dnetlib.data.mdstore.backends.MockBackend ;
2023-02-09 13:36:26 +01:00
import eu.dnetlib.data.mdstore.model.MDStore ;
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion ;
2023-02-10 10:27:09 +01:00
import eu.dnetlib.data.mdstore.model.MDStoreType ;
2023-02-09 13:36:26 +01:00
import eu.dnetlib.data.mdstore.model.MDStoreVersion ;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo ;
2023-02-10 14:54:37 +01:00
import eu.dnetlib.data.mdstore.model.MetadataRecord ;
2023-02-06 14:50:10 +01:00
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository ;
import eu.dnetlib.data.mdstore.repository.MDStoreRepository ;
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository ;
import eu.dnetlib.data.mdstore.repository.MDStoreWithInfoRepository ;
2023-02-06 15:53:07 +01:00
import eu.dnetlib.errors.MDStoreManagerException ;
2023-02-06 14:50:10 +01:00
2023-02-10 10:27:09 +01:00
@Service
public class MDStoreService {
2023-02-06 14:50:10 +01:00
@Autowired
private MDStoreRepository mdstoreRepository ;
@Autowired
private MDStoreVersionRepository mdstoreVersionRepository ;
@Autowired
private MDStoreCurrentVersionRepository mdstoreCurrentVersionRepository ;
@Autowired
private MDStoreWithInfoRepository mdstoreWithInfoRepository ;
@Autowired
2023-02-09 13:36:26 +01:00
protected JdbcTemplate jdbcTemplate ;
2023-02-06 14:50:10 +01:00
2023-02-10 10:27:09 +01:00
@Autowired
private HdfsBackend hdfsBackend ;
@Autowired
2023-02-10 14:54:37 +01:00
private MockBackend mockBackend ;
@Autowired
2023-02-10 10:27:09 +01:00
private DefaultBackend defaultBackend ;
2023-02-06 15:53:07 +01:00
private static final Logger log = LoggerFactory . getLogger ( MDStoreService . class ) ;
2023-02-06 14:50:10 +01:00
2023-02-08 12:23:53 +01:00
public List < MDStoreWithInfo > listMdStores ( ) {
return StreamSupport . stream ( mdstoreWithInfoRepository . findAll ( ) . spliterator ( ) , false )
. sorted ( Comparator . comparing ( ( Function < MDStoreWithInfo , String > ) md - > md . getDatasourceName ( ) ) . thenComparing ( md - > md . getId ( ) ) )
. collect ( Collectors . toList ( ) ) ;
2023-02-06 14:50:10 +01:00
}
public List < String > listMdStoreIDs ( ) {
2023-02-08 12:23:53 +01:00
return mdstoreRepository . findAll ( ) . stream ( ) . map ( MDStore : : getId ) . sorted ( ) . collect ( Collectors . toList ( ) ) ;
2023-02-06 14:50:10 +01:00
}
public long countMdStores ( ) {
return mdstoreRepository . count ( ) ;
}
public Iterable < MDStoreVersion > listVersions ( final String mdId ) {
2023-02-08 12:23:53 +01:00
return mdstoreVersionRepository . findByMdstoreOrderById ( mdId ) ;
2023-02-06 14:50:10 +01:00
}
public List < String > listExpiredVersions ( ) {
return jdbcTemplate
. queryForList ( " select v.id from mdstore_versions v left outer join mdstore_current_versions cv on (v.id = cv.current_version) where v.writing = false and v.readcount = 0 and cv.mdstore is null; " , String . class ) ;
}
public MDStoreWithInfo findMdStore ( final String mdId ) throws MDStoreManagerException {
return mdstoreWithInfoRepository . findById ( mdId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Missing mdstore: " + mdId ) ) ;
}
public MDStoreVersion findVersion ( final String versionId ) throws MDStoreManagerException {
return mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Missing mdstore version: " + versionId ) ) ;
}
@Transactional
public String createMDStore ( final String format ,
final String layout ,
final String interpretation ,
2023-02-10 10:27:09 +01:00
final MDStoreType type ,
2023-02-06 14:50:10 +01:00
final String dsName ,
final String dsId ,
final String apiId ) {
2023-02-10 10:27:09 +01:00
final MDStore md = newMDStore ( format , layout , interpretation , type , dsName , dsId , apiId , apiId ) ;
2023-02-06 14:50:10 +01:00
mdstoreRepository . save ( md ) ;
2023-02-10 10:27:09 +01:00
final MDStoreVersion v = newMDStoreVersion ( md , false ) ;
2023-02-06 14:50:10 +01:00
mdstoreVersionRepository . save ( v ) ;
mdstoreCurrentVersionRepository . save ( MDStoreCurrentVersion . newInstance ( v ) ) ;
return md . getId ( ) ;
}
2023-02-10 10:27:09 +01:00
private MDStoreVersion newMDStoreVersion ( final MDStore md , final boolean writing ) {
final MDStoreVersion v = new MDStoreVersion ( ) ;
2023-02-27 11:43:28 +01:00
final LocalDateTime now = LocalDateTime . now ( ) ;
2023-02-09 13:36:26 +01:00
2023-02-27 11:43:28 +01:00
final String versionId = md . getId ( ) + " - " + now . toEpochSecond ( ZoneOffset . UTC ) ;
2023-02-10 10:27:09 +01:00
v . setId ( versionId ) ;
v . setMdstore ( md . getId ( ) ) ;
v . setLastUpdate ( null ) ;
v . setWriting ( writing ) ;
v . setReadCount ( 0 ) ;
v . setSize ( 0 ) ;
v . setLastUpdate ( now ) ;
selectBackend ( md . getType ( ) ) . completeNewMDStoreVersion ( v ) ;
return v ;
}
2023-02-09 13:36:26 +01:00
2023-02-06 14:50:10 +01:00
@Transactional
2023-02-07 11:22:20 +01:00
public void deleteMdStore ( final String mdId ) throws MDStoreManagerException {
2023-02-06 14:50:10 +01:00
2023-02-10 10:27:09 +01:00
final MDStore md = mdstoreRepository . findById ( mdId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " MDStore not found: " + mdId ) ) ;
2023-02-06 14:50:10 +01:00
if ( mdstoreVersionRepository . countByMdstoreAndReadCountGreaterThan ( mdId , 0 ) > 0 ) {
log . error ( " Read transactions found on mdstore: " + mdId ) ;
throw new MDStoreManagerException ( " Read transactions found on mdstore: " + mdId ) ;
}
if ( mdstoreVersionRepository . countByMdstoreAndWriting ( mdId , true ) > 0 ) {
log . error ( " Write transactions found on mdstore: " + mdId ) ;
throw new MDStoreManagerException ( " Write transactions found on mdstore: " + mdId ) ;
}
mdstoreCurrentVersionRepository . deleteById ( mdId ) ;
mdstoreVersionRepository . deleteByMdstore ( mdId ) ;
mdstoreRepository . deleteById ( mdId ) ;
2023-02-10 10:27:09 +01:00
selectBackend ( md . getType ( ) ) . delete ( md ) ;
2023-02-06 14:50:10 +01:00
}
@Transactional
public MDStoreVersion startReading ( final String mdId ) throws MDStoreManagerException {
final MDStoreCurrentVersion cv =
mdstoreCurrentVersionRepository . findById ( mdId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Missing mdstore: " + mdId ) ) ;
final MDStoreVersion v = mdstoreVersionRepository . findById ( cv . getCurrentVersion ( ) )
. orElseThrow ( ( ) - > new MDStoreManagerException ( " Missing version: " + cv . getCurrentVersion ( ) ) ) ;
v . setReadCount ( v . getReadCount ( ) + 1 ) ;
mdstoreVersionRepository . save ( v ) ;
return v ;
}
@Transactional
public MDStoreVersion endReading ( final String versionId ) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
v . setReadCount ( Math . max ( 0 , v . getReadCount ( ) - 1 ) ) ;
return v ;
}
@Transactional
public MDStoreVersion resetReading ( final String versionId ) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
v . setReadCount ( 0 ) ;
return v ;
}
@Transactional
2023-02-10 10:27:09 +01:00
public MDStoreVersion prepareMdStoreVersion ( final String mdId ) throws MDStoreManagerException {
final MDStore md = mdstoreRepository . findById ( mdId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " MDStore not found " ) ) ;
final MDStoreVersion v = newMDStoreVersion ( md , true ) ;
2023-02-06 14:50:10 +01:00
mdstoreVersionRepository . save ( v ) ;
return v ;
}
@Transactional
public MDStoreVersion commitMdStoreVersion ( final String versionId , final long size ) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Invalid version: " + versionId ) ) ;
mdstoreCurrentVersionRepository . save ( MDStoreCurrentVersion . newInstance ( v ) ) ;
v . setWriting ( false ) ;
v . setSize ( size ) ;
2023-02-27 11:43:28 +01:00
v . setLastUpdate ( LocalDateTime . now ( ) ) ;
2023-02-06 14:50:10 +01:00
mdstoreVersionRepository . save ( v ) ;
return v ;
}
2023-02-09 13:36:26 +01:00
public synchronized void deleteExpiredVersions ( ) {
2023-02-07 11:22:20 +01:00
log . info ( " Deleting expired version... " ) ;
for ( final String versionId : listExpiredVersions ( ) ) {
try {
deleteMdStoreVersion ( versionId , true ) ;
} catch ( final MDStoreManagerException e ) {
log . warn ( " Error deleteting version " + versionId , e ) ;
}
}
log . info ( " Done. " ) ;
}
2023-02-06 14:50:10 +01:00
@Transactional
2023-02-07 11:22:20 +01:00
public void deleteMdStoreVersion ( final String versionId , final boolean force ) throws MDStoreManagerException {
2023-02-06 14:50:10 +01:00
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
2023-02-10 10:27:09 +01:00
final MDStore md = mdstoreRepository . findById ( v . getMdstore ( ) ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
2023-02-06 14:50:10 +01:00
if ( mdstoreCurrentVersionRepository
. countByCurrentVersion ( versionId ) > 0 ) {
throw new MDStoreManagerException ( " I cannot delete this version because it is the current version " ) ;
}
if ( ! force ) {
if ( v . isWriting ( ) ) { throw new MDStoreManagerException ( " I cannot delete this version because it is in write mode " ) ; }
if ( v . getReadCount ( ) > 0 ) { throw new MDStoreManagerException ( " I cannot delete this version because it is in read mode " ) ; }
}
mdstoreVersionRepository . delete ( v ) ;
2023-02-10 10:27:09 +01:00
selectBackend ( md . getType ( ) ) . delete ( v ) ;
}
public Set < String > listVersionInternalFiles ( final String versionId ) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
final MDStore md = mdstoreRepository . findById ( v . getMdstore ( ) ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " MDStore not found " ) ) ;
return selectBackend ( md . getType ( ) ) . listInternalFiles ( v ) ;
}
2023-02-10 14:54:37 +01:00
public List < MetadataRecord > listVersionRecords ( final String versionId , final long limit ) throws MDStoreManagerException {
2023-02-10 10:27:09 +01:00
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
final MDStore md = mdstoreRepository . findById ( v . getMdstore ( ) ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " MDStore not found " ) ) ;
return selectBackend ( md . getType ( ) ) . listEntries ( v , limit ) ;
}
2023-04-18 16:13:42 +02:00
public Stream < MetadataRecord > streamVersionRecords ( final String versionId ) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository . findById ( versionId ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " Version not found " ) ) ;
final MDStore md = mdstoreRepository . findById ( v . getMdstore ( ) ) . orElseThrow ( ( ) - > new MDStoreManagerException ( " MDStore not found " ) ) ;
return selectBackend ( md . getType ( ) ) . streamEntries ( v ) ;
}
2023-02-10 10:27:09 +01:00
public MDStore newMDStore (
final String format ,
final String layout ,
final String interpretation ,
final MDStoreType type ,
final String dsName ,
final String dsId ,
final String apiId ,
final String hdfsBasePath ) {
final String mdId = " md- " + UUID . randomUUID ( ) ;
final MDStore md = new MDStore ( ) ;
md . setId ( mdId ) ;
md . setFormat ( format ) ;
md . setLayout ( layout ) ;
md . setType ( type ) ;
md . setInterpretation ( interpretation ) ;
2023-02-27 11:43:28 +01:00
md . setCreationDate ( LocalDateTime . now ( ) ) ;
2023-02-10 10:27:09 +01:00
md . setDatasourceName ( dsName ) ;
md . setDatasourceId ( dsId ) ;
md . setApiId ( apiId ) ;
selectBackend ( type ) . completeNewMDStore ( md ) ;
return md ;
}
public Map < MDStoreType , Set < String > > fixInconsistencies ( final boolean delete ) throws MDStoreManagerException {
final Map < MDStoreType , Set < String > > res = new LinkedHashMap < > ( ) ;
res . put ( MDStoreType . HDFS , hdfsBackend . fixInconsistencies ( delete ) ) ;
2023-02-10 14:54:37 +01:00
res . put ( MDStoreType . MOCK , mockBackend . fixInconsistencies ( delete ) ) ;
2023-02-10 10:27:09 +01:00
// TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE
return res ;
}
private MDStoreBackend selectBackend ( final MDStoreType type ) {
switch ( type ) {
case HDFS :
return hdfsBackend ;
2023-02-10 14:54:37 +01:00
case MOCK :
return mockBackend ;
2023-02-10 10:27:09 +01:00
default :
return defaultBackend ;
}
2023-02-06 14:50:10 +01:00
}
}