MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Adding migration for reference middle tables

This commit is contained in:
Thomas Georgios Giannos 2023-11-24 11:36:43 +02:00
parent e4801f94ed
commit a6d175dcb3
9 changed files with 436 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetDataRepositoryDao.java Normal file
View File

@ -0,0 +1,10 @@
package eu.old.eudat.data.dao.entities;
import eu.old.eudat.data.dao.DatabaseAccessLayer;
import eu.old.eudat.data.entities.DatasetDataRepository;
import java.util.UUID;
public interface DatasetDataRepositoryDao extends DatabaseAccessLayer<DatasetDataRepository, UUID> {
}

51
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetDataRepositoryDaoImpl.java Normal file
View File

@ -0,0 +1,51 @@
package eu.old.eudat.data.dao.entities;
import eu.old.eudat.data.dao.DatabaseAccess;
import eu.old.eudat.data.dao.databaselayer.service.DatabaseService;
import eu.old.eudat.data.entities.DatasetDataRepository;
import eu.old.eudat.queryable.QueryableList;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@Component
public class DatasetDataRepositoryDaoImpl extends DatabaseAccess<DatasetDataRepository> implements DatasetDataRepositoryDao{
public DatasetDataRepositoryDaoImpl(DatabaseService<DatasetDataRepository> databaseService) {
super(databaseService);
}
@Override
public DatasetDataRepository createOrUpdate(DatasetDataRepository item) {
return this.getDatabaseService().createOrUpdate(item,DatasetDataRepository.class);
}
@Override
public CompletableFuture<DatasetDataRepository> createOrUpdateAsync(DatasetDataRepository item) {
return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
}
@Override
@Async
public DatasetDataRepository find(UUID id) {
return getDatabaseService().getQueryable(DatasetDataRepository.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
}
@Override
public DatasetDataRepository find(UUID id, String hint) {
throw new UnsupportedOperationException();
}
@Override
public void delete(DatasetDataRepository item) {
this.getDatabaseService().delete(item);
}
@Override
public QueryableList<DatasetDataRepository> asQueryable() {
return this.getDatabaseService().getQueryable(DatasetDataRepository.class);
}
}

10
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetRegistryDao.java Normal file
View File

@ -0,0 +1,10 @@
package eu.old.eudat.data.dao.entities;
import eu.old.eudat.data.dao.DatabaseAccessLayer;
import eu.old.eudat.data.entities.DatasetRegistry;
import java.util.UUID;
public interface DatasetRegistryDao extends DatabaseAccessLayer<DatasetRegistry, UUID> {
}

51
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetRegistryDaoImpl.java Normal file
View File

@ -0,0 +1,51 @@
package eu.old.eudat.data.dao.entities;
import eu.old.eudat.data.dao.DatabaseAccess;
import eu.old.eudat.data.dao.databaselayer.service.DatabaseService;
import eu.old.eudat.data.entities.DatasetRegistry;
import eu.old.eudat.queryable.QueryableList;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@Component
public class DatasetRegistryDaoImpl extends DatabaseAccess<DatasetRegistry> implements DatasetRegistryDao {
public DatasetRegistryDaoImpl(DatabaseService<DatasetRegistry> databaseService) {
super(databaseService);
}
@Override
public DatasetRegistry createOrUpdate(DatasetRegistry item) {
return this.getDatabaseService().createOrUpdate(item,DatasetRegistry.class);
}
@Override
public CompletableFuture<DatasetRegistry> createOrUpdateAsync(DatasetRegistry item) {
return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
}
@Override
@Async
public DatasetRegistry find(UUID id) {
return getDatabaseService().getQueryable(DatasetRegistry.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
}
@Override
public DatasetRegistry find(UUID id, String hint) {
throw new UnsupportedOperationException();
}
@Override
public void delete(DatasetRegistry item) {
this.getDatabaseService().delete(item);
}
@Override
public QueryableList<DatasetRegistry> asQueryable() {
return this.getDatabaseService().getQueryable(DatasetRegistry.class);
}
}

43
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/entities/DatasetRegistry.java
View File

@ -1,17 +1,19 @@
package eu.old.eudat.data.entities;
import eu.old.eudat.queryable.queryableentity.DataEntity;
import org.hibernate.annotations.GenericGenerator;
import org.hibernate.annotations.Type;
import jakarta.persistence.*;
import java.io.Serializable;
import java.util.List;
import java.util.UUID;
@Entity
@Table(name = "\"DatasetRegistry\"")
public class DatasetRegistry {
public class DatasetRegistry implements DataEntity<DatasetRegistry, UUID> {
@Id
@GeneratedValue
@ -19,13 +21,13 @@ public class DatasetRegistry {
@Column(name = "\"ID\"", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
private UUID id;
//DEPWARN dependency to Hibernate and PostgreSQL
@Column(name = "\"Dataset\"", nullable = false)
private UUID dataset;
@ManyToOne
@JoinColumn(name = "\"Dataset\"", nullable = false)
private Dataset dataset;
//DEPWARN dependency to Hibernate and PostgreSQL
@Column(name = "\"Registry\"", nullable = false)
private UUID registry;
@ManyToOne
@JoinColumn(name = "\"Registry\"", nullable = false)
private Registry registry;
@Column(name = "\"Role\"")
private Integer role;
@ -41,19 +43,19 @@ public class DatasetRegistry {
this.id = id;
}
public UUID getDataset() {
public Dataset getDataset() {
return dataset;
}
public void setDataset(UUID dataset) {
public void setDataset(Dataset dataset) {
this.dataset = dataset;
}
public UUID getRegistry() {
public Registry getRegistry() {
return registry;
}
public void setRegistry(UUID registry) {
public void setRegistry(Registry registry) {
this.registry = registry;
}
@ -72,4 +74,23 @@ public class DatasetRegistry {
public void setData(String data) {
this.data = data;
}
@Override
public void update(DatasetRegistry entity) {
this.dataset = entity.getDataset();
this.registry = entity.getRegistry();
this.role = entity.getRole();
}
@Override
public UUID getKeys() {
return this.id;
}
@Override
public DatasetRegistry buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
String currentBase = base.isEmpty() ? "" : base + ".";
if(fields.contains(currentBase + "id")) this.id = UUID.fromString((String) tuple.get(0).get(currentBase + "id"));
return this;
}
}

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/logic/services/operations/DatabaseRepository.java
View File

@ -48,8 +48,12 @@ public interface DatabaseRepository {
DatasetExternalDatasetDao getDatasetExternalDatasetDao();
DatasetDataRepositoryDao getDatasetDataRepositoryDao();
DatasetServiceDao getDatasetServiceDao();
DatasetRegistryDao getDatasetRegistryDao();
EmailConfirmationDao getLoginConfirmationEmailDao();
ProjectDao getProjectDao();

22
dmp-migration-tool/web/src/main/java/eu/old/eudat/logic/services/operations/DatabaseRepositoryImpl.java
View File

@ -33,7 +33,9 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
private ContentDao contentDao;
private DMPProfileDao dmpProfileDao;
private DatasetExternalDatasetDao datasetExternalDatasetDao;
private DatasetDataRepositoryDao datasetDataRepositoryDao;
private DatasetServiceDao datasetServiceDao;
private DatasetRegistryDao datasetRegistryDao;
private EmailConfirmationDao loginConfirmationEmailDao;
private ProjectDao projectDao;
private FunderDao funderDao;
@ -250,6 +252,16 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
this.datasetExternalDatasetDao = datasetExternalDatasetDao;
}
@Override
public DatasetDataRepositoryDao getDatasetDataRepositoryDao() {
return datasetDataRepositoryDao;
}
@Autowired
public void setDatasetDataRepositoryDao(DatasetDataRepositoryDao datasetDataRepositoryDao) {
this.datasetDataRepositoryDao = datasetDataRepositoryDao;
}
@Override
public DatasetServiceDao getDatasetServiceDao() {
return datasetServiceDao;
@ -260,6 +272,16 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
this.datasetServiceDao = datasetServiceDao;
}
@Override
public DatasetRegistryDao getDatasetRegistryDao() {
return datasetRegistryDao;
}
@Autowired
public void setDatasetRegistryDao(DatasetRegistryDao datasetRegistryDao) {
this.datasetRegistryDao = datasetRegistryDao;
}
@Override
public EmailConfirmationDao getLoginConfirmationEmailDao() {
return loginConfirmationEmailDao;

237
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceMigrationService.java Normal file
View File

@ -0,0 +1,237 @@
package eu.old.eudat.migration;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DescriptionReferenceEntity;
import eu.eudat.data.DmpReferenceEntity;
import eu.old.eudat.data.dao.entities.*;
import eu.old.eudat.data.entities.*;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
@Service
public class ReferenceMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
private final DatabaseRepository databaseRepository;
private final QueryFactory queryFactory;
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final EntityManager entityManager;
public ReferenceMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, EntityManager entityManager) {
this.databaseRepository = databaseRepository;
this.queryFactory = queryFactory;
this.entityManager = entityManager;
}
public void migrateDmpReferences() {
migrateDmpOrganizations();
migrateDmpResearchers();
}
public void migrateDatasetReferences() {
migrateDatasetDataRepositories();
migrateDatasetExternalDatasets();
migrateDatasetRegistries();
migrateDatasetServices();
}
public void migrateDmpOrganizations() {
OrganisationDao organisationDao = databaseRepository.getOrganisationDao();
long total = organisationDao.asQueryable().count();
logger.debug("Migrate Dmp Organisation (from Organization) Total : " + total);
int page = 0;
List<Organisation> items;
do {
items = organisationDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Organisation " + page * PageSize + " of " + total);
for (Organisation item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getDmps()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDmpResearchers() {
ResearcherDao researcherDao = databaseRepository.getResearcherDao();
long total = researcherDao.asQueryable().count();
logger.debug("Migrate Dmp Researcher (from Researcher) Total : " + total);
int page = 0;
List<Researcher> items;
do {
items = researcherDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Researcher " + page * PageSize + " of " + total);
for (Researcher item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getdMPs()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDatasetDataRepositories() {
DatasetDataRepositoryDao datasetDataRepositoryDao = databaseRepository.getDatasetDataRepositoryDao();
long total = datasetDataRepositoryDao.asQueryable().count();
logger.debug("Migrate Dataset DataRepository (from DatasetDataRepository) Total : " + total);
int page = 0;
List<DatasetDataRepository> items;
do {
items = datasetDataRepositoryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset DataRepository " + page * PageSize + " of " + total);
for (DatasetDataRepository item : items) {
// entityManager.detach(item);
DescriptionReferenceEntity data = new DescriptionReferenceEntity();
data.setId(UUID.randomUUID());
data.setDescriptionId(item.getDataset().getId());
data.setReferenceId(item.getDataRepository().getId());
data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDatasetExternalDatasets() {
DatasetExternalDatasetDao datasetExternalDatasetDao = databaseRepository.getDatasetExternalDatasetDao();
long total = datasetExternalDatasetDao.asQueryable().count();
logger.debug("Migrate Dataset ExternalDataset (from DatasetExternalDataset) Total : " + total);
int page = 0;
List<DatasetExternalDataset> items;
do {
items = datasetExternalDatasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset ExternalDataset " + page * PageSize + " of " + total);
for (DatasetExternalDataset item : items) {
// entityManager.detach(item);
DescriptionReferenceEntity data = new DescriptionReferenceEntity();
data.setId(UUID.randomUUID());
data.setDescriptionId(item.getDataset().getId());
data.setReferenceId(item.getExternalDataset().getId());
data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDatasetRegistries() {
DatasetRegistryDao datasetRegistryDao = databaseRepository.getDatasetRegistryDao();
long total = datasetRegistryDao.asQueryable().count();
logger.debug("Migrate Dataset Registry (from DatasetRegistry) Total : " + total);
int page = 0;
List<DatasetRegistry> items;
do {
items = datasetRegistryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset Registry " + page * PageSize + " of " + total);
for (DatasetRegistry item : items) {
// entityManager.detach(item);
DescriptionReferenceEntity data = new DescriptionReferenceEntity();
data.setId(UUID.randomUUID());
data.setDescriptionId(item.getDataset().getId());
data.setReferenceId(item.getRegistry().getId());
data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDatasetServices() {
DatasetServiceDao datasetServiceDao = databaseRepository.getDatasetServiceDao();
long total = datasetServiceDao.asQueryable().count();
logger.debug("Migrate Dataset Service (from DatasetService) Total : " + total);
int page = 0;
List<DatasetService> items;
do {
items = datasetServiceDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset Service " + page * PageSize + " of " + total);
for (DatasetService item : items) {
// entityManager.detach(item);
DescriptionReferenceEntity data = new DescriptionReferenceEntity();
data.setId(UUID.randomUUID());
data.setDescriptionId(item.getDataset().getId());
data.setReferenceId(item.getService().getId());
data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
}

24
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -32,6 +32,7 @@ public class MigrationController {
private final DatasetMigrationService datasetMigrationService;
private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService;
private final OrganizationMigrationService organizationMigrationService;
private final ReferenceMigrationService referenceMigrationService;
public MigrationController(
DmpMigrationService dmpMigrationService,
@ -45,8 +46,8 @@ public class MigrationController {
RegistryMigrationService registryMigrationService,
ResearcherMigrationService researcherMigrationService,
ServiceMigrationService serviceMigrationService,
OrganizationMigrationService organizationMigrationService
) {
OrganizationMigrationService organizationMigrationService,
ReferenceMigrationService referenceMigrationService) {
this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -59,6 +60,7 @@ public class MigrationController {
this.researcherMigrationService = researcherMigrationService;
this.serviceMigrationService = serviceMigrationService;
this.organizationMigrationService = organizationMigrationService;
this.referenceMigrationService = referenceMigrationService;
}
@GetMapping("dmps")
@ -79,11 +81,9 @@ public class MigrationController {
@Transactional
public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.dmpDatasetProfileMigrationService.migrate();
return true;
}
@GetMapping("references")
@Transactional
public boolean migrateReferences() {
@ -97,7 +97,21 @@ public class MigrationController {
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
return true;
}
}
@GetMapping("dmp-references")
@Transactional
public boolean migrateDmpReferences() {
this.referenceMigrationService.migrateDmpReferences();
return true;
}
@GetMapping("dataset-references")
@Transactional
public boolean migrateDatasetReferences() {
this.referenceMigrationService.migrateDatasetReferences();
return true;
}
@GetMapping("organizations")
@Transactional