MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Complete migration process regarding Dataset Profiles and DMPs-Datasets

This commit is contained in:
Konstantinos Triantafyllou 2022-05-21 00:30:56 +03:00
parent e127e353de
commit 5275062b39
13 changed files with 439 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dmp-backend/roadmap/src/main/java/eu/eudat/migration/DMPRoadmapMigration.java
View File

@ -26,12 +26,19 @@ public class DMPRoadmapMigration implements CommandLineRunner {
public static void main(String[] args) {
new SpringApplicationBuilder(DMPRoadmapMigration.class)
.web(WebApplicationType.NONE).run(args);
System.out.println("Migration Finished");
}
@Override
public void run(String... args) throws Exception {
System.out.println("Logging in as administrator");
this.service.login();
System.out.println("Migration Started");
this.service.migrate();
System.out.println(this.service.getDatesetProfiles());
System.out.println("Migration Finished. Logging out");
Thread.sleep(20000);
this.service.clear();
this.service.logout();
System.out.println("Done");
}
}

9
dmp-backend/roadmap/src/main/java/eu/eudat/migration/dao/TemplateRepository.java
View File

@ -2,8 +2,17 @@ package eu.eudat.migration.dao;
import eu.eudat.migration.entities.Template;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import java.util.List;
@ConditionalOnProperty(prefix = "roadmap", name="database.url")
public interface TemplateRepository extends JpaRepository<Template, Long> {
@Query(value = "SELECT DISTINCT (t.familyId) from Template t")
List<Long> findAllGroups();
List<Template> findAllByFamilyId(Long familyId, Sort sort);
}

159
dmp-backend/roadmap/src/main/java/eu/eudat/migration/entities/Plan.java
View File

@ -1,18 +1,21 @@
package eu.eudat.migration.entities;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.models.data.admin.composite.DatasetProfile;
import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.dmp.AssociatedProfile;
import eu.eudat.models.data.dmp.DataManagementPlan;
import eu.eudat.models.data.dmp.DataManagementPlanEditorModel;
import eu.eudat.models.data.dmp.Organisation;
import eu.eudat.models.data.funder.FunderDMPEditorModel;
import eu.eudat.models.data.grant.GrantDMPEditorModel;
import eu.eudat.models.data.project.ProjectDMPEditorModel;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import org.hibernate.annotations.LazyCollection;
import org.hibernate.annotations.LazyCollectionOption;
import javax.persistence.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
@Entity
@Table(name = "plans")
@ -86,6 +89,34 @@ public class Plan {
DENIED
}
@Override
public String toString() {
return "Plan{" +
"id=" + id +
", title='" + title + '\'' +
", createdAt=" + createdAt +
", updatedAt=" + updatedAt +
", identifier='" + identifier + '\'' +
", description='" + description + '\'' +
", visibility=" + visibility +
", feedbackRequested=" + feedbackRequested +
", complete=" + complete +
", organization=" + organization +
", funder=" + funder +
", startDate=" + startDate +
", endDate=" + endDate +
", grantId=" + grantId +
", apiClient=" + apiClient +
", researchDomain=" + researchDomain +
", ethicalIssues=" + ethicalIssues +
", ethicalIssuesDescription='" + ethicalIssuesDescription + '\'' +
", ethicalIssuesReport='" + ethicalIssuesReport + '\'' +
", fundingStatus=" + fundingStatus +
", users=" + users +
", answers=" + answers +
'}';
}
public Long getId() {
return id;
}
@ -262,31 +293,125 @@ public class Plan {
this.answers = answers;
}
public DataManagementPlanEditorModel buildDMP(DatasetProfile datasetProfile) {
public DataManagementPlanEditorModel buildDMP(UUID profileId, String profileLabel,
eu.eudat.migration.properties.Entity funder,
eu.eudat.migration.properties.Entity grant) {
DataManagementPlanEditorModel dmpModel = new DataManagementPlanEditorModel();
dmpModel.setAssociatedUsers(new ArrayList<>());
dmpModel.setDatasets(new ArrayList<>());
dmpModel.setDatasetsToBeFinalized(new ArrayList<>());
dmpModel.setDescription(getDescription());
dmpModel.setDynamicFields(new ArrayList<>());
dmpModel.setExtraProperties(buildExtraProperties());
dmpModel.setFunder(buildFunder(funder));
dmpModel.setGrant(buildGrant(grant));
dmpModel.setLabel(getTitle());
dmpModel.setOrganisations(buildOrganizations());
dmpModel.setProfiles(buildProfiles(profileId, profileLabel));
dmpModel.setProject(buildProject());
dmpModel.setResearchers(new ArrayList<>());
dmpModel.setStatus(0);
return dmpModel;
}
public DatasetWizardModel buildDataset(DatasetProfile datasetProfile) {
public Map<String, Object> buildExtraProperties() {
Map<String, Object> properties = new HashMap<>();
properties.put("costs", new ArrayList<>());
properties.put("language", "en");
properties.put("publicDate", getCreatedAt());
properties.put("visible", getVisibility() == Visibility.PUBLIC);
return properties;
}
public FunderDMPEditorModel buildFunder(eu.eudat.migration.properties.Entity defaultFunder) {
FunderDMPEditorModel funder = new FunderDMPEditorModel();
if(getFunder() != null) {
funder.setLabel(getFunder().getName());
funder.setReference(getFunder().getAbbreviation());
} else {
funder.setLabel(defaultFunder.getLabel());
funder.setReference(defaultFunder.getReference());
}
return funder;
}
public GrantDMPEditorModel buildGrant(eu.eudat.migration.properties.Entity defaultGrant) {
GrantDMPEditorModel grant = new GrantDMPEditorModel();
if(getGrantId() != null) {
grant.setLabel(getGrantId().toString());
grant.setReference(getGrantId().toString());
grant.setDescription(getGrantId().toString());
} else {
grant.setLabel(defaultGrant.getLabel());
grant.setReference(defaultGrant.getReference());
grant.setDescription(defaultGrant.getDescription());
}
return grant;
}
public List<Organisation> buildOrganizations() {
if(getOrganization() != null) {
Organisation organisation = new Organisation();
organisation.setLabel(getOrganization().getName());
organisation.setReference("Internal:" + getOrganization().getAbbreviation());
organisation.setKey("Internal");
return Collections.singletonList(organisation);
} else {
return new ArrayList<>();
}
}
public List<AssociatedProfile> buildProfiles(UUID id, String label) {
AssociatedProfile profile = new AssociatedProfile();
profile.setId(id);
profile.setLabel(label);
return Collections.singletonList(profile);
}
public ProjectDMPEditorModel buildProject() {
return new ProjectDMPEditorModel();
}
public DatasetWizardModel buildDataset(UUID profileId, String profileLabel, DataManagementPlan dmp, PagedDatasetProfile pagedDatasetProfile) {
DatasetWizardModel datasetModel = new DatasetWizardModel();
datasetModel.setDataRepositories(new ArrayList<>());
if(datasetProfile != null) {
/* TODO Dataset Definition */
PagedDatasetProfile datasetProfileDefinition = new PagedDatasetProfile();
datasetModel.setDatasetProfileDefinition(datasetProfileDefinition);
/* TODO DMP */
DatasetProfileOverviewModel profile = new DatasetProfileOverviewModel();
profile.setId(datasetProfile.getId());
profile.setLabel(datasetProfile.getLabel());
datasetModel.setProfile(profile);
}
buildPagedDatasetProfile(pagedDatasetProfile);
datasetModel.setDatasetProfileDefinition(pagedDatasetProfile);
datasetModel.setDescription(this.getDescription());
datasetModel.setDmp(dmp);
datasetModel.setExternalDatasets(new ArrayList<>());
datasetModel.setLabel(this.getTitle());
DatasetProfileOverviewModel profile = new DatasetProfileOverviewModel();
profile.setId(profileId);
profile.setLabel(profileLabel);
datasetModel.setProfile(profile);
datasetModel.setRegistries(new ArrayList<>());
datasetModel.setServices(new ArrayList<>());
datasetModel.setTags(new ArrayList<>());
return datasetModel;
}
public void buildPagedDatasetProfile(PagedDatasetProfile pagedDatasetProfile) {
pagedDatasetProfile.getPages().forEach(page -> {
page.getSections().forEach(section -> {
section.getCompositeFields().forEach(compositeField -> {
compositeField.getFields().forEach(field -> {
Long questionId = Question.fieldsQuestionsMap.get(field.getId());
if( questionId != null) {
Optional<Answer> optionalAnswer = answers.stream().filter(answer -> answer.getQuestion().getId().equals(questionId)).findFirst();
optionalAnswer.ifPresent(answer -> {
if(answer.getQuestion().getQuestionFormat().getOptionBased()) {
/* TODO check with real data */
field.setValue(answer.getText());
} else {
field.setValue(answer.getText());
}
});
}
});
});
});
});
}
}

3
dmp-backend/roadmap/src/main/java/eu/eudat/migration/entities/Question.java
View File

@ -14,6 +14,8 @@ import java.util.stream.Collectors;
@Table(name = "questions")
public class Question {
public static Map<String, Long> fieldsQuestionsMap = new HashMap<>();
@Id
private Long id;
private String text;
@ -204,6 +206,7 @@ public class Question {
field.setVisible(this.buildVisibility());
field.setViewStyle(this.getQuestionFormat().buildViewStyle());
field.setData(this.buildData());
fieldsQuestionsMap.put(field.getId(), getId());
return Collections.singletonList(field);
}

5
dmp-backend/roadmap/src/main/java/eu/eudat/migration/entities/RoadmapSection.java
View File

@ -6,10 +6,7 @@ import org.hibernate.annotations.LazyCollection;
import org.hibernate.annotations.LazyCollectionOption;
import javax.persistence.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
@Entity

2
dmp-backend/roadmap/src/main/java/eu/eudat/migration/entities/Template.java
View File

@ -230,7 +230,7 @@ public class Template {
datasetProfile.setDescription(this.getDescription());
datasetProfile.setLanguage(this.getLocale());
datasetProfile.setStatus((short) 1);
datasetProfile.setVersion((short) 0);
datasetProfile.setVersion(getVersion());
datasetProfile.setUsers(new ArrayList<>());
this.buildPages(datasetProfile);
return datasetProfile;

32
dmp-backend/roadmap/src/main/java/eu/eudat/migration/properties/Argos.java Normal file
View File

@ -0,0 +1,32 @@
package eu.eudat.migration.properties;
public class Argos {
private String url;
private String username;
private String password;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}

15
dmp-backend/roadmap/src/main/java/eu/eudat/migration/properties/ConfigProperties.java
View File

@ -5,7 +5,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "roadmap")
public class ConfigProperties {
private Database database;
private String argos;
private Argos argos;
private DefaultEntities entities;
public Database getDatabase() {
return database;
@ -15,11 +16,19 @@ public class ConfigProperties {
this.database = database;
}
public String getArgos() {
public Argos getArgos() {
return argos;
}
public void setArgos(String argos) {
public void setArgos(Argos argos) {
this.argos = argos;
}
public DefaultEntities getEntities() {
return entities;
}
public void setEntities(DefaultEntities entities) {
this.entities = entities;
}
}

22
dmp-backend/roadmap/src/main/java/eu/eudat/migration/properties/DefaultEntities.java Normal file
View File

@ -0,0 +1,22 @@
package eu.eudat.migration.properties;
public class DefaultEntities {
private Entity funder;
private Entity grant;
public Entity getFunder() {
return funder;
}
public void setFunder(Entity funder) {
this.funder = funder;
}
public Entity getGrant() {
return grant;
}
public void setGrant(Entity grant) {
this.grant = grant;
}
}

31
dmp-backend/roadmap/src/main/java/eu/eudat/migration/properties/Entity.java Normal file
View File

@ -0,0 +1,31 @@
package eu.eudat.migration.properties;
public class Entity {
public String reference;
public String label;
public String description;
public String getReference() {
return reference;
}
public void setReference(String reference) {
this.reference = reference;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

114
dmp-backend/roadmap/src/main/java/eu/eudat/migration/services/ArgosService.java
View File

@ -1,23 +1,135 @@
package eu.eudat.migration.services;
import eu.eudat.migration.properties.Argos;
import eu.eudat.migration.properties.ConfigProperties;
import eu.eudat.models.data.admin.composite.DatasetProfile;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.dmp.DataManagementPlan;
import eu.eudat.models.data.dmp.DataManagementPlanEditorModel;
import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.models.data.login.Credentials;
import eu.eudat.models.data.principal.PrincipalModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.util.Objects;
import java.util.UUID;
@Service
public class ArgosService {
private final String argos;
private final Argos argos;
private UUID token;
private final RestTemplate restTemplate = new RestTemplate();
@Autowired
public ArgosService(ConfigProperties properties) {
this.argos = properties.getArgos();
}
/**
* Auth methods
* */
public void login() {
Credentials credentials = new Credentials();
credentials.setUsername(argos.getUsername());
credentials.setSecret(argos.getPassword());
HttpEntity<Credentials> request = new HttpEntity<>(credentials, setHeaders());
PrincipalModel principal = Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/auth/nativelogin",
HttpMethod.POST, request, new ParameterizedTypeReference<ResponseItem<PrincipalModel>>() {
}).getBody()).getPayload();
this.token = principal.getToken();
}
public void logout() {
HttpEntity<Object> request = new HttpEntity<>(null, setHeaders());
this.restTemplate.exchange(argos.getUrl() + "/api/auth/logout", HttpMethod.POST, request, new ParameterizedTypeReference<ResponseItem<Principal>>() {});
this.token = null;
}
/**
* Dataset Profiles
* */
public UUID createDatasetProfile(DatasetProfile datasetProfile) {
HttpEntity<DatasetProfile> request = new HttpEntity<>(datasetProfile, setHeaders());
return Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/admin/addDmp",
HttpMethod.POST, request, new ParameterizedTypeReference<UUID>() {
}).getBody());
}
public UUID createNewVersionDatasetProfile(UUID id, DatasetProfile datasetProfile) {
HttpEntity<DatasetProfile> request = new HttpEntity<>(datasetProfile, setHeaders());
return Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/admin/newVersion/" + id,
HttpMethod.POST, request, new ParameterizedTypeReference<UUID>() {
}).getBody());
}
public void deleteDatasetProfile(UUID id) {
HttpEntity<Object> request = new HttpEntity<>(null, setHeaders());
this.restTemplate.exchange(argos.getUrl() + "/api/admin/" + id, HttpMethod.DELETE, request, Object.class);
}
/**
* DMP methods
* */
public DataManagementPlan createDMP(DataManagementPlanEditorModel model) {
HttpEntity<DataManagementPlanEditorModel> request = new HttpEntity<>(model, setHeaders());
return Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/dmps/",
HttpMethod.POST, request, new ParameterizedTypeReference<ResponseItem<DataManagementPlan>>() {
}).getBody()).getPayload();
}
public void deleteDMP(UUID id) {
HttpEntity<Object> request = new HttpEntity<>(null, setHeaders());
this.restTemplate.exchange(argos.getUrl() + "/api/dmps/" + id, HttpMethod.DELETE, request, Object.class);
}
/**
* Dataset Methods
* */
public PagedDatasetProfile getDataset(UUID id) {
HttpEntity<Object> request = new HttpEntity<>(null, setHeaders());
return Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/datasets/get/" + id,
HttpMethod.GET, request, new ParameterizedTypeReference<ResponseItem<PagedDatasetProfile>>() {
}).getBody()).getPayload();
}
public DatasetWizardModel createDataset(DatasetWizardModel model) {
HttpEntity<DatasetWizardModel> request = new HttpEntity<>(model, setHeaders());
return Objects.requireNonNull(this.restTemplate.exchange(argos.getUrl() + "/api/datasets/",
HttpMethod.POST, request, new ParameterizedTypeReference<ResponseItem<DatasetWizardModel>>() {
}).getBody()).getPayload();
}
public void deleteDataset(UUID id) {
HttpEntity<Object> request = new HttpEntity<>(null, setHeaders());
this.restTemplate.exchange(argos.getUrl() + "/api/datasets/delete/" + id, HttpMethod.DELETE, request, Object.class);
}
public HttpHeaders setHeaders() {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
if(this.token != null) {
headers.add("AuthToken", this.token.toString());
}
return headers;
}
public UUID getToken() {
return token;
}
public void setToken(UUID token) {
this.token = token;
}
}

73
dmp-backend/roadmap/src/main/java/eu/eudat/migration/services/DMPRoadmapService.java
View File

@ -2,18 +2,21 @@ package eu.eudat.migration.services;
import eu.eudat.migration.dao.TemplateRepository;
import eu.eudat.migration.dao.UserRepository;
import eu.eudat.migration.entities.Plan;
import eu.eudat.migration.entities.Template;
import eu.eudat.migration.entities.User;
import eu.eudat.migration.properties.ConfigProperties;
import eu.eudat.migration.properties.DefaultEntities;
import eu.eudat.models.data.admin.composite.DatasetProfile;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.dmp.DataManagementPlan;
import eu.eudat.models.data.dmp.DataManagementPlanEditorModel;
import eu.eudat.models.data.principal.PrincipalModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@Service
@ConditionalOnProperty(prefix = "roadmap", name = "database.url")
@ -21,35 +24,65 @@ public class DMPRoadmapService {
/** DMPRoadmap Repositories*/
private final TemplateRepository templateRepository;
private final UserRepository userRepository;
private final List<DatasetProfile> datasetProfiles = new ArrayList<>();
private final List<DataManagementPlanEditorModel> dmps = new ArrayList<>();
private final List<DatasetWizardModel> datasets = new ArrayList<>();
/** Default Entities*/
private final DefaultEntities defaultEntities;
/** Services */
private final ArgosService argosService;
/** Metadata */
private final Map<Long, UUID> groups = new HashMap<>();
private final List<UUID> datasetProfiles = new ArrayList<>();
private final List<UUID> dmps = new ArrayList<>();
private final List<UUID> datasets = new ArrayList<>();
@Autowired
public DMPRoadmapService(TemplateRepository templateRepository, UserRepository userRepository) {
public DMPRoadmapService(TemplateRepository templateRepository, UserRepository userRepository, ArgosService argosService, ConfigProperties properties) {
this.templateRepository = templateRepository;
this.userRepository = userRepository;
this.defaultEntities = properties.getEntities();
this.argosService = argosService;
}
public void login() {
this.argosService.login();
}
public void logout() {
this.argosService.logout();
}
public void migrate() {
List<Template> templates = templateRepository.findAll();
List<User> users = userRepository.findAll();
templates.forEach(template -> {
DatasetProfile datasetProfile = template.buildDatasetProfile();
this.datasetProfiles.add(datasetProfile);
/*template.getPlans().forEach(plan -> {
DataManagementPlanEditorModel dmpModel = this.buildDMP(plan, null);
this.dmps.add(dmpModel);
});*/
});
/*users.forEach(user -> {
try {
templateRepository.findAllGroups().forEach(group -> {
List<Template> templates = templateRepository.findAllByFamilyId(group, Sort.by(Sort.Direction.ASC, "version"));
AtomicReference<UUID> id = new AtomicReference<>(null);
templates.forEach(template -> {
DatasetProfile datasetProfile = template.buildDatasetProfile();
if (id.get() == null) {
id.set(this.argosService.createDatasetProfile(datasetProfile));
} else {
id.set(this.argosService.createNewVersionDatasetProfile(id.get(), datasetProfile));
}
this.datasetProfiles.add(id.get());
template.getPlans().forEach(plan -> {
DataManagementPlanEditorModel dataManagementPlanEditorModel = plan.buildDMP(id.get(), datasetProfile.getLabel(),
defaultEntities.getFunder(), defaultEntities.getGrant());
DataManagementPlan dmp = this.argosService.createDMP(dataManagementPlanEditorModel);
this.dmps.add(dmp.getId());
DatasetWizardModel dataset = plan.buildDataset(id.get(), datasetProfile.getLabel(), dmp, this.argosService.getDataset(id.get()));
dataset = this.argosService.createDataset(dataset);
this.datasets.add(dataset.getId());
});
});
});
} catch (Exception e) {
System.out.println("Migration Failed. All data will be deleted.");
this.clear();
this.logout();
}
}
});*/
public void clear() {
this.datasets.forEach(this.argosService::deleteDataset);
this.dmps.forEach(this.argosService::deleteDMP);
this.datasetProfiles.forEach(this.argosService::deleteDatasetProfile);
}
}

12
dmp-backend/roadmap/src/main/resources/application.properties
View File

@ -8,4 +8,14 @@ roadmap.database.dialect=org.hibernate.dialect.MySQL5Dialect
#roadmap.database.dialect=org.hibernate.dialect.PostgreSQL92Dialect
## Argos Backend URL ##
roadmap.argos=localhost:8081
roadmap.argos.url=http://localhost:8081
roadmap.argos.username=root
roadmap.argos.password=root
## Default entities ##
roadmap.entities.funder.reference=no_funder
roadmap.entities.funder.label=No funder
roadmap.entities.grant.reference=no_grant
roadmap.entities.grant.label=No grant
roadmap.entities.grant.description=No grant available