package eu.dnetlib.data.mdstore; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import javax.transaction.Transactional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import eu.dnetlib.data.mdstore.backends.DefaultBackend; import eu.dnetlib.data.mdstore.backends.HdfsBackend; import eu.dnetlib.data.mdstore.backends.MDStoreBackend; import eu.dnetlib.data.mdstore.backends.MockBackend; import eu.dnetlib.data.mdstore.model.MDStore; import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion; import eu.dnetlib.data.mdstore.model.MDStoreType; import eu.dnetlib.data.mdstore.model.MDStoreVersion; import eu.dnetlib.data.mdstore.model.MDStoreWithInfo; import eu.dnetlib.data.mdstore.model.MetadataRecord; import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository; import eu.dnetlib.data.mdstore.repository.MDStoreRepository; import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository; import eu.dnetlib.data.mdstore.repository.MDStoreWithInfoRepository; import eu.dnetlib.errors.MDStoreManagerException; @Service public class MDStoreService { @Autowired private MDStoreRepository mdstoreRepository; @Autowired private MDStoreVersionRepository mdstoreVersionRepository; @Autowired private MDStoreCurrentVersionRepository mdstoreCurrentVersionRepository; @Autowired private MDStoreWithInfoRepository mdstoreWithInfoRepository; @Autowired protected JdbcTemplate jdbcTemplate; @Autowired private HdfsBackend hdfsBackend; @Autowired private MockBackend mockBackend; @Autowired private DefaultBackend defaultBackend; private static final Logger log = LoggerFactory.getLogger(MDStoreService.class); public List listMdStores() { return StreamSupport.stream(mdstoreWithInfoRepository.findAll().spliterator(), false) .sorted(Comparator.comparing((Function) md -> md.getDatasourceName()).thenComparing(md -> md.getId())) .collect(Collectors.toList()); } public List listMdStoreIDs() { return mdstoreRepository.findAll().stream().map(MDStore::getId).sorted().collect(Collectors.toList()); } public long countMdStores() { return mdstoreRepository.count(); } public Iterable listVersions(final String mdId) { return mdstoreVersionRepository.findByMdstoreOrderById(mdId); } public List listExpiredVersions() { return jdbcTemplate .queryForList("select v.id from mdstore_versions v left outer join mdstore_current_versions cv on (v.id = cv.current_version) where v.writing = false and v.readcount = 0 and cv.mdstore is null;", String.class); } public MDStoreWithInfo findMdStore(final String mdId) throws MDStoreManagerException { return mdstoreWithInfoRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore: " + mdId)); } public MDStoreVersion findVersion(final String versionId) throws MDStoreManagerException { return mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore version: " + versionId)); } @Transactional public String createMDStore(final String format, final String layout, final String interpretation, final MDStoreType type, final String dsName, final String dsId, final String apiId) { final MDStore md = newMDStore(format, layout, interpretation, type, dsName, dsId, apiId, apiId); mdstoreRepository.save(md); final MDStoreVersion v = newMDStoreVersion(md, false); mdstoreVersionRepository.save(v); mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v)); return md.getId(); } private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) { final MDStoreVersion v = new MDStoreVersion(); final LocalDateTime now = LocalDateTime.now(); final String versionId = md.getId() + "-" + now.toEpochSecond(ZoneOffset.UTC); v.setId(versionId); v.setMdstore(md.getId()); v.setLastUpdate(null); v.setWriting(writing); v.setReadCount(0); v.setSize(0); v.setLastUpdate(now); selectBackend(md.getType()).completeNewMDStoreVersion(v); return v; } @Transactional public void deleteMdStore(final String mdId) throws MDStoreManagerException { final MDStore md = mdstoreRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("MDStore not found: " + mdId)); if (mdstoreVersionRepository.countByMdstoreAndReadCountGreaterThan(mdId, 0) > 0) { log.error("Read transactions found on mdstore: " + mdId); throw new MDStoreManagerException("Read transactions found on mdstore: " + mdId); } if (mdstoreVersionRepository.countByMdstoreAndWriting(mdId, true) > 0) { log.error("Write transactions found on mdstore: " + mdId); throw new MDStoreManagerException("Write transactions found on mdstore: " + mdId); } mdstoreCurrentVersionRepository.deleteById(mdId); mdstoreVersionRepository.deleteByMdstore(mdId); mdstoreRepository.deleteById(mdId); selectBackend(md.getType()).delete(md); } @Transactional public MDStoreVersion startReading(final String mdId) throws MDStoreManagerException { final MDStoreCurrentVersion cv = mdstoreCurrentVersionRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore: " + mdId)); final MDStoreVersion v = mdstoreVersionRepository.findById(cv.getCurrentVersion()) .orElseThrow(() -> new MDStoreManagerException("Missing version: " + cv.getCurrentVersion())); v.setReadCount(v.getReadCount() + 1); mdstoreVersionRepository.save(v); return v; } @Transactional public MDStoreVersion endReading(final String versionId) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); v.setReadCount(Math.max(0, v.getReadCount() - 1)); return v; } @Transactional public MDStoreVersion resetReading(final String versionId) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); v.setReadCount(0); return v; } @Transactional public MDStoreVersion prepareMdStoreVersion(final String mdId) throws MDStoreManagerException { final MDStore md = mdstoreRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("MDStore not found")); final MDStoreVersion v = newMDStoreVersion(md, true); mdstoreVersionRepository.save(v); return v; } @Transactional public MDStoreVersion commitMdStoreVersion(final String versionId, final long size) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Invalid version: " + versionId)); mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v)); v.setWriting(false); v.setSize(size); v.setLastUpdate(LocalDateTime.now()); mdstoreVersionRepository.save(v); return v; } public synchronized void deleteExpiredVersions() { log.info("Deleting expired version..."); for (final String versionId : listExpiredVersions()) { try { deleteMdStoreVersion(versionId, true); } catch (final MDStoreManagerException e) { log.warn("Error deleteting version " + versionId, e); } } log.info("Done."); } @Transactional public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("Version not found")); if (mdstoreCurrentVersionRepository .countByCurrentVersion(versionId) > 0) { throw new MDStoreManagerException("I cannot delete this version because it is the current version"); } if (!force) { if (v.isWriting()) { throw new MDStoreManagerException("I cannot delete this version because it is in write mode"); } if (v.getReadCount() > 0) { throw new MDStoreManagerException("I cannot delete this version because it is in read mode"); } } mdstoreVersionRepository.delete(v); selectBackend(md.getType()).delete(v); } public Set listVersionInternalFiles(final String versionId) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found")); return selectBackend(md.getType()).listInternalFiles(v); } public List listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found")); return selectBackend(md.getType()).listEntries(v, limit); } public Stream streamVersionRecords(final String versionId) throws MDStoreManagerException { final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found")); final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found")); return selectBackend(md.getType()).streamEntries(v); } public MDStore newMDStore( final String format, final String layout, final String interpretation, final MDStoreType type, final String dsName, final String dsId, final String apiId, final String hdfsBasePath) { final String mdId = "md-" + UUID.randomUUID(); final MDStore md = new MDStore(); md.setId(mdId); md.setFormat(format); md.setLayout(layout); md.setType(type); md.setInterpretation(interpretation); md.setCreationDate(LocalDateTime.now()); md.setDatasourceName(dsName); md.setDatasourceId(dsId); md.setApiId(apiId); selectBackend(type).completeNewMDStore(md); return md; } public Map> fixInconsistencies(final boolean delete) throws MDStoreManagerException { final Map> res = new LinkedHashMap<>(); res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete)); res.put(MDStoreType.MOCK, mockBackend.fixInconsistencies(delete)); // TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE return res; } private MDStoreBackend selectBackend(final MDStoreType type) { switch (type) { case HDFS: return hdfsBackend; case MOCK: return mockBackend; default: return defaultBackend; } } }