MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Migrating tags from elastic service

This commit is contained in:
Thomas Georgios Giannos 2023-12-14 17:37:43 +02:00
parent f405fdc32a
commit a213ca3e84
3 changed files with 112 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dmp-migration-tool/elastic/src/main/java/eu/old/eudat/elastic/repository/DatasetRepository.java
View File

@ -88,10 +88,10 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
searchSourceBuilder.query(nestedQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
return Arrays.stream(response.getHits().getHits())
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
.map(SearchHits::getHits).flatMap(Arrays::stream)
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).findFirst().orElse(null);
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class))).findFirst().orElse(null);
// GetRequest request = new GetRequest("datasets", id);
// GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
// return new Dataset().fromElasticEntity(response.getSourceAsMap());

99
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java Normal file
View File

@ -0,0 +1,99 @@
package eu.old.eudat.migration;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.DescriptionTagEntity;
import eu.eudat.data.TagEntity;
import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.elastic.entities.Tag;
import eu.old.eudat.elastic.repository.DatasetRepository;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant;
import java.util.*;
@Service
public class TagMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(TagMigrationService.class));
private final DatabaseRepository databaseRepository;
private final QueryFactory queryFactory;
private final DatasetRepository datasetRepository;
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final EntityManager entityManager;
public TagMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, DatasetRepository datasetRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository;
this.queryFactory = queryFactory;
this.datasetRepository = datasetRepository;
this.entityManager = entityManager;
}
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
DatasetDao datasetDao = databaseRepository.getDatasetDao();
long total = datasetDao.asQueryable().count();
logger.debug("Migrate Tags for Dataset Total : " + total);
int page = 0;
Set<String> savedTagNames = new HashSet<>();
Map<String, UUID> savedTagIdsByName = new HashMap<>();
List<Dataset> items;
do {
items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total);
for (Dataset item : items) {
eu.old.eudat.elastic.entities.Dataset elasticDataset = this.datasetRepository.findDocument(item.getId().toString());
if (elasticDataset == null) {
logger.error("No dataset with id {} found on elastic search. Skipping tag migration for this dataset", item.getId());
continue;
}
boolean tagAlreadyExists;
if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) {
for(Tag tag : elasticDataset.getTags()) {
tagAlreadyExists = savedTagNames.contains(tag.getName());
savedTagNames.add(tag.getName());
if (!tagAlreadyExists) {
TagEntity tagEntity = new TagEntity();
tagEntity.setId(UUID.fromString(tag.getId()));
tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now());
tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(tag.getName(), tagEntity.getId());
this.entityManager.persist(tagEntity);
logger.debug("New tag saved '{}'", tag.getName());
}
DescriptionTagEntity descriptionTagEntity = new DescriptionTagEntity();
descriptionTagEntity.setId(UUID.randomUUID());
descriptionTagEntity.setTagId(savedTagIdsByName.get(tag.getName()));
descriptionTagEntity.setDescriptionId(item.getId());
descriptionTagEntity.setCreatedAt(Instant.now());
descriptionTagEntity.setUpdatedAt(Instant.now());
descriptionTagEntity.setIsActive(IsActive.Active);
this.entityManager.persist(descriptionTagEntity);
}
}
this.entityManager.flush();
}
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
}
}

12
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -30,6 +30,7 @@ public class MigrationController {
private final ResearcherMigrationService researcherMigrationService;
private final ServiceMigrationService serviceMigrationService;
private final DatasetMigrationService datasetMigrationService;
private final TagMigrationService tagMigrationService;
private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService;
private final OrganizationMigrationService organizationMigrationService;
private final ReferenceMigrationService referenceMigrationService;
@ -48,7 +49,7 @@ public class MigrationController {
ProjectMigrationService projectMigrationService,
RegistryMigrationService registryMigrationService,
ResearcherMigrationService researcherMigrationService,
ServiceMigrationService serviceMigrationService,
ServiceMigrationService serviceMigrationService, TagMigrationService tagMigrationService,
OrganizationMigrationService organizationMigrationService,
ReferenceMigrationService referenceMigrationService,
UserContactInfoMigrationService userContactInfoMigrationService,
@ -64,6 +65,7 @@ public class MigrationController {
this.registryMigrationService = registryMigrationService;
this.researcherMigrationService = researcherMigrationService;
this.serviceMigrationService = serviceMigrationService;
this.tagMigrationService = tagMigrationService;
this.organizationMigrationService = organizationMigrationService;
this.referenceMigrationService = referenceMigrationService;
this.userContactInfoMigrationService = userContactInfoMigrationService;
@ -87,6 +89,7 @@ public class MigrationController {
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate();
this.tagMigrationService.migrate();
this.dmpUserMigrationService.migrate();
@ -129,6 +132,13 @@ public class MigrationController {
return true;
}
@GetMapping("tags")
@Transactional
public boolean migrateTags() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.tagMigrationService.migrate();
return true;
}
@GetMapping("dmp-dataset-profiles")
@Transactional
public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {