From 2b2b0cad0a6d0f975d9bf61339d4fe51c8f26699 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 27 Feb 2023 11:43:28 +0100 Subject: [PATCH 01/19] first import of msro --- .../main/java/eu/dnetlib/MainDBConfig.java | 4 +- .../is/importer/OldProfilesImporter.java | 4 +- .../is/importer/WfHistoryImporter.java | 14 +- .../manager/wf/WfHistoryAjaxController.java | 34 +++ .../msro/history/WfHistoryAjaxController.java | 49 ---- .../dnetlib/data/mdstore/MDStoreService.java | 11 +- .../data/mdstore/backends/MockBackend.java | 6 +- .../dnetlib/data/mdstore/model/MDStore.java | 8 +- .../data/mdstore/model/MDStoreVersion.java | 8 +- .../data/mdstore/model/MDStoreWithInfo.java | 14 +- .../java/eu/dnetlib/dsm/model/Datasource.java | 19 +- .../eu/dnetlib/dsm/model/Organization.java | 8 +- .../errors/WorkflowManagerException.java | 14 + .../is/model/resource/SimpleResource.java | 14 +- .../manager/wf/model/EmailTemplate.java | 50 ++++ .../manager/wf/model/GraphArcDbEntry.java | 28 ++ .../manager/wf/model/GraphNodeDbEntry.java | 64 +++++ .../wf/model/GraphParameterDbEntry.java | 39 +++ .../wf/model/GraphParameterValueDbEntry.java | 45 +++ .../wf/model/NotificationCondition.java | 5 + .../wf/model}/WfProcessExecution.java | 27 +- .../manager/wf/model/WorkflowDbEntry.java | 83 ++++++ .../manager/wf/model/WorkflowInstance.java | 195 +++++++++++++ .../manager/wf/model/WorkflowParamDesc.java | 5 + .../wf/model/WorkflowSubscription.java | 68 +++++ .../wf/model/WorkflowSubscriptionPK.java | 39 +++ .../main/java/eu/dnetlib/utils/DateUtils.java | 20 +- .../main/java/eu/dnetlib/utils/Stoppable.java | 10 + .../eu/dnetlib/utils/StoppableDetails.java | 33 +++ .../src/main/resources/sql/schema.sql | 73 ++++- libs/dnet-is-services/pom.xml | 5 + .../main/java/eu/dnetlib/dsm/DsmService.java | 4 +- .../eu/dnetlib/dsm/domain/ApiDetails.java | 20 +- .../dnetlib/dsm/domain/DatasourceDetails.java | 32 +-- .../dsm/domain/DatasourceDetailsUpdate.java | 14 +- .../domain/DatasourceIgnoredProperties.java | 14 +- .../dsm/domain/DatasourceSnippetExtended.java | 20 +- .../domain/OrganizationIgnoredProperties.java | 8 +- .../dsm/repository/DatasourceRepository.java | 4 +- .../eu/dnetlib/dsm/utils/DsmMappingUtils.java | 4 +- .../is/resource/SimpleResourceService.java | 4 +- .../eu/dnetlib/manager/wf/WorkflowLogger.java | 51 ++++ .../repository/EmailTemplateRepository.java | 9 + .../WfProcessExecutionRepository.java | 16 ++ .../repository/WorkflowDbEntryRepository.java | 9 + .../WorkflowInstanceRepository.java | 9 + .../WorkflowSubscriptionRepository.java | 13 + .../WfProcessExecutionRepository.java | 13 - .../notifications/mail/EmailDispatcher.java | 147 ++++++++++ libs/dnet-wf-service/pom.xml | 48 ++++ .../java/eu/dnetlib/manager/wf/NodeInfo.java | 26 ++ .../java/eu/dnetlib/manager/wf/WfManager.java | 20 ++ .../wf/annotations/StreamMimeType.java | 8 + .../manager/wf/annotations/WfNode.java | 20 ++ .../wf/annotations/WfNodeOperation.java | 11 + .../manager/wf/annotations/WfParam.java | 8 + .../wf/cron/ScheduledWorkflowLauncher.java | 138 ++++++++++ .../manager/wf/nodes/AbstractJobNode.java | 45 +++ .../manager/wf/nodes/AsyncJobNode.java | 28 ++ .../manager/wf/nodes/CollectOAINode.java | 29 ++ .../manager/wf/nodes/DefaultJobNode.java | 21 ++ .../wf/nodes/LaunchWorkflowJobNode.java | 78 ++++++ .../dnetlib/manager/wf/nodes/NodeStatus.java | 8 + .../dnetlib/manager/wf/nodes/ProcessNode.java | 39 +++ .../manager/wf/nodes/SimpleJobNode.java | 12 + .../dnetlib/manager/wf/nodes/SuccessNode.java | 17 ++ .../manager/wf/notification/EmailSender.java | 46 ++++ .../manager/wf/workflows/graph/Arc.java | 35 +++ .../manager/wf/workflows/graph/Graph.java | 80 ++++++ .../wf/workflows/graph/GraphLoader.java | 180 ++++++++++++ .../manager/wf/workflows/graph/GraphNode.java | 149 ++++++++++ .../workflows/graph/GraphNodeParameter.java | 56 ++++ .../manager/wf/workflows/procs/Env.java | 54 ++++ .../wf/workflows/procs/ProcessAware.java | 10 + .../wf/workflows/procs/ProcessEngine.java | 203 ++++++++++++++ .../wf/workflows/procs/ProcessFactory.java | 67 +++++ .../wf/workflows/procs/ProcessRegistry.java | 132 +++++++++ .../manager/wf/workflows/procs/Token.java | 147 ++++++++++ .../wf/workflows/procs/WorkflowExecutor.java | 155 +++++++++++ .../wf/workflows/procs/WorkflowProcess.java | 258 ++++++++++++++++++ .../manager/wf/workflows/util/NodeHelper.java | 66 +++++ .../wf/workflows/util/NodeTokenCallback.java | 13 + .../wf/workflows/util/ProcessCallback.java | 12 + .../wf/workflows/util/ProgressProvider.java | 6 + .../util/SubWorkflowProgressProvider.java | 37 +++ .../util/ValidNodeValuesFetcher.java | 71 +++++ .../wf/workflows/util/WorkflowsConstants.java | 119 ++++++++ libs/pom.xml | 1 + pom.xml | 6 +- 89 files changed, 3645 insertions(+), 201 deletions(-) create mode 100644 apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java delete mode 100644 apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java rename libs/dnet-is-common/src/main/java/eu/dnetlib/{msro/model/history => manager/wf/model}/WfProcessExecution.java (81%) create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java delete mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java create mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java create mode 100644 libs/dnet-wf-service/pom.xml create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java index 0a5cb752..5978f4ac 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java @@ -20,7 +20,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration @EnableTransactionManagement @EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = { - "eu.dnetlib.is", "eu.dnetlib.data.mdstore", "eu.dnetlib.msro" + "eu.dnetlib.is", "eu.dnetlib.data.mdstore", "eu.dnetlib.manager" }) public class MainDBConfig { @@ -38,7 +38,7 @@ public class MainDBConfig { @Qualifier("mainDataSource") final DataSource ds) { return builder .dataSource(ds) - .packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.data.mdstore.model") + .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model") .persistenceUnit("is") .build(); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java index 58a395d8..e37519b2 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java @@ -1,6 +1,6 @@ package eu.dnetlib.is.importer; -import java.util.Date; +import java.time.LocalDateTime; import javax.transaction.Transactional; @@ -47,7 +47,7 @@ public class OldProfilesImporter { final Document doc = DocumentHelper.parseText(xml); final String id = StringUtils.substringBefore(doc.valueOf("//RESOURCE_IDENTIFIER/@value"), "_"); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); final SimpleResource res = new SimpleResource(); res.setId(id); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java index 95d1d6c2..a1eaa4b0 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java @@ -1,11 +1,13 @@ package eu.dnetlib.is.importer; import java.io.File; -import java.util.Date; +import java.time.Instant; +import java.time.LocalDateTime; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.TimeZone; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -17,8 +19,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository; -import eu.dnetlib.msro.model.history.WfProcessExecution; +import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.wf.repository.WfProcessExecutionRepository; @Service public class WfHistoryImporter { @@ -53,8 +55,10 @@ public class WfHistoryImporter { wf.setDsApi(node.get("dataprovider:interface").asText()); } - wf.setStartDate(new Date(NumberUtils.toLong(node.get("system:startDate").asText()))); - wf.setEndDate(new Date(NumberUtils.toLong(node.get("system:endDate").asText()))); + wf.setStartDate(LocalDateTime + .ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:startDate").asText())), TimeZone.getDefault().toZoneId())); + + wf.setEndDate(LocalDateTime.ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:endDate").asText())), TimeZone.getDefault().toZoneId())); if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) { wf.setStatus("success"); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java new file mode 100644 index 00000000..cccf4db3 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java @@ -0,0 +1,34 @@ +package eu.dnetlib.manager.wf; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.manager.wf.model.WfProcessExecution; + +@RestController +@RequestMapping("/ajax/wfs") +public class WfHistoryAjaxController { + + @Autowired + private WorkflowLogger logger; + + @GetMapping("/") + public List history( + @RequestParam(required = true) final int total, + @RequestParam(required = false) final Long from, + @RequestParam(required = false) final Long to) { + return logger.history(total, from, to); + } + + @GetMapping("/{processId}") + public WfProcessExecution getProcessExecution(@PathVariable final String processId) { + return logger.getProcessExecution(processId); + } + +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java deleted file mode 100644 index a6ccc190..00000000 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java +++ /dev/null @@ -1,49 +0,0 @@ -package eu.dnetlib.msro.history; - -import java.util.Date; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository; -import eu.dnetlib.msro.model.history.WfProcessExecution; - -@RestController -@RequestMapping("/ajax/wfs") -public class WfHistoryAjaxController { - - @Autowired - private WfProcessExecutionRepository wfProcessExecutionRepository; - - @Deprecated - public static final int MAX_NUMBER_OF_RECENT_WFS = 100; - - @GetMapping("/") - public List history( - @RequestParam(required = true) final int total, - @RequestParam(required = false) final Long from, - @RequestParam(required = false) final Long to) { - if (from == null && to == null) { - return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList(); - } else if (from == null) { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(0), new Date(to)); - } else if (to == null) { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date()); - } else { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date(to)); - } - } - - @GetMapping("/{processId}") - public WfProcessExecution getProcess(@PathVariable final String processId) { - return wfProcessExecutionRepository.findById(processId).get(); - } - -} diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java index db7f48b1..41adb299 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java @@ -1,7 +1,8 @@ package eu.dnetlib.data.mdstore; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.Comparator; -import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -112,9 +113,9 @@ public class MDStoreService { private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) { final MDStoreVersion v = new MDStoreVersion(); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); - final String versionId = md.getId() + "-" + now.getTime(); + final String versionId = md.getId() + "-" + now.toEpochSecond(ZoneOffset.UTC); v.setId(versionId); v.setMdstore(md.getId()); v.setLastUpdate(null); @@ -189,7 +190,7 @@ public class MDStoreService { mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v)); v.setWriting(false); v.setSize(size); - v.setLastUpdate(new Date()); + v.setLastUpdate(LocalDateTime.now()); mdstoreVersionRepository.save(v); return v; @@ -258,7 +259,7 @@ public class MDStoreService { md.setLayout(layout); md.setType(type); md.setInterpretation(interpretation); - md.setCreationDate(new Date()); + md.setCreationDate(LocalDateTime.now()); md.setDatasourceName(dsName); md.setDatasourceId(dsId); md.setApiId(apiId); diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java index 66ba910b..5b1a1d84 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java @@ -1,8 +1,8 @@ package eu.dnetlib.data.mdstore.backends; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -49,8 +49,8 @@ public class MockBackend implements MDStoreBackend { rec.setOriginalId("mck-" + i); rec.setId("mock________::mck-" + i); rec.setBody("" + i + ""); - rec.setDateOfCollection(new Date().getTime()); - rec.setDateOfTransformation(new Date().getTime()); + rec.setDateOfCollection(Instant.now().toEpochMilli()); + rec.setDateOfTransformation(Instant.now().toEpochMilli()); rec.setEncoding("XML"); rec.setProvenance(MOCK_PROVENANCE); list.add(rec); diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java index 82548da9..aaf488da 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -56,7 +56,7 @@ public class MDStore implements Serializable { @Column(name = "creation_date") @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; + private LocalDateTime creationDate; @Override public int hashCode() { @@ -143,11 +143,11 @@ public class MDStore implements Serializable { this.params = params; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java index d2e07c08..4ba17ffa 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -37,7 +37,7 @@ public class MDStoreVersion implements Serializable { @Column(name = "lastupdate") @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; + private LocalDateTime lastUpdate; @Column(name = "size") private long size = 0; @@ -78,11 +78,11 @@ public class MDStoreVersion implements Serializable { this.readCount = readCount; } - public Date getLastUpdate() { + public LocalDateTime getLastUpdate() { return lastUpdate; } - public void setLastUpdate(final Date lastUpdate) { + public void setLastUpdate(final LocalDateTime lastUpdate) { this.lastUpdate = lastUpdate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java index 17bdb4f8..b95133f1 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -55,11 +55,11 @@ public class MDStoreWithInfo implements Serializable { @Column(name = "creation_date") @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; + private LocalDateTime creationDate; @Column(name = "lastupdate") @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; + private LocalDateTime lastUpdate; @Column(name = "size") private long size = 0; @@ -143,19 +143,19 @@ public class MDStoreWithInfo implements Serializable { this.currentVersion = currentVersion; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } - public Date getLastUpdate() { + public LocalDateTime getLastUpdate() { return lastUpdate; } - public void setLastUpdate(final Date lastUpdate) { + public void setLastUpdate(final LocalDateTime lastUpdate) { this.lastUpdate = lastUpdate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java index 36f357c4..fe882b1a 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java @@ -2,6 +2,7 @@ package eu.dnetlib.dsm.model; import java.io.Serializable; import java.sql.Date; +import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; @@ -89,7 +90,7 @@ public class Datasource implements Serializable { private String registeredby; - private Date registrationdate; + private LocalDateTime registrationdate; private String subjects; @@ -99,10 +100,10 @@ public class Datasource implements Serializable { private Boolean consentTermsOfUse; @Column(name = "consenttermsofusedate") - private Date consentTermsOfUseDate; + private LocalDateTime consentTermsOfUseDate; @Column(name = "lastconsenttermsofusedate") - private Date lastConsentTermsOfUseDate; + private LocalDateTime lastConsentTermsOfUseDate; @Column(name = "fulltextdownload") private Boolean fullTextDownload; @@ -401,11 +402,11 @@ public class Datasource implements Serializable { this.registeredby = registeredby; } - public Date getRegistrationdate() { + public LocalDateTime getRegistrationdate() { return registrationdate; } - public void setRegistrationdate(final Date registrationdate) { + public void setRegistrationdate(final LocalDateTime registrationdate) { this.registrationdate = registrationdate; } @@ -433,19 +434,19 @@ public class Datasource implements Serializable { this.consentTermsOfUse = consentTermsOfUse; } - public Date getConsentTermsOfUseDate() { + public LocalDateTime getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public void setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public void setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; } - public Date getLastConsentTermsOfUseDate() { + public LocalDateTime getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public void setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public void setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java index ec5c9e87..25ee8533 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java @@ -1,7 +1,7 @@ package eu.dnetlib.dsm.model; import java.io.Serializable; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.Set; import javax.persistence.CascadeType; @@ -34,7 +34,7 @@ public class Organization implements Serializable { private String country; private String collectedfrom; - private Date dateofcollection; + private LocalDateTime dateofcollection; private String provenanceaction; @ManyToMany(cascade = { @@ -98,11 +98,11 @@ public class Organization implements Serializable { this.collectedfrom = collectedfrom; } - public Date getDateofcollection() { + public LocalDateTime getDateofcollection() { return dateofcollection; } - public void setDateofcollection(final Date dateofcollection) { + public void setDateofcollection(final LocalDateTime dateofcollection) { this.dateofcollection = dateofcollection; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java new file mode 100644 index 00000000..bc29ade1 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java @@ -0,0 +1,14 @@ +package eu.dnetlib.errors; + +public class WorkflowManagerException extends Exception { + + private static final long serialVersionUID = -9067581185191425823L; + + public WorkflowManagerException(final String message, final Throwable cause) { + super(message, cause); + } + + public WorkflowManagerException(final String message) { + super(message); + } +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java index 60565e0b..a2ec6c79 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java @@ -1,7 +1,7 @@ package eu.dnetlib.is.model.resource; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import javax.persistence.Column; import javax.persistence.Entity; @@ -31,11 +31,11 @@ public class SimpleResource implements Serializable { @Temporal(TemporalType.TIMESTAMP) @Column(name = "creation_date") - private Date creationDate; + private LocalDateTime creationDate; @Temporal(TemporalType.TIMESTAMP) @Column(name = "modification_date") - private Date modificationDate; + private LocalDateTime modificationDate; public String getId() { return id; @@ -69,19 +69,19 @@ public class SimpleResource implements Serializable { this.description = description; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } - public Date getModificationDate() { + public LocalDateTime getModificationDate() { return modificationDate; } - public void setModificationDate(final Date modificationDate) { + public void setModificationDate(final LocalDateTime modificationDate) { this.modificationDate = modificationDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java new file mode 100644 index 00000000..0ba91984 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java @@ -0,0 +1,50 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "emails") +public class EmailTemplate implements Serializable { + + private static final long serialVersionUID = -8424437958140000626L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "subject") + private String subject; + + @Column(name = "message") + private String message; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getSubject() { + return subject; + } + + public void setSubject(final String subject) { + this.subject = subject; + } + + public String getMessage() { + return message; + } + + public void setMessage(final String message) { + this.message = message; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java new file mode 100644 index 00000000..5ec1447f --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java @@ -0,0 +1,28 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +public class GraphArcDbEntry implements Serializable { + + private static final long serialVersionUID = 7866138976929522262L; + + private String name; + private String to; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getTo() { + return to; + } + + public void setTo(final String to) { + this.to = to; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java new file mode 100644 index 00000000..ca1e0f91 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java @@ -0,0 +1,64 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +public class GraphNodeDbEntry implements Serializable { + + private static final long serialVersionUID = -3695762832959801906L; + + private String name; + private String type; + private boolean isStart = false; + private boolean isJoin = false; + private GraphArcDbEntry[] arcs; + private GraphParameterDbEntry[] params; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public boolean isStart() { + return isStart; + } + + public void setStart(final boolean isStart) { + this.isStart = isStart; + } + + public boolean isJoin() { + return isJoin; + } + + public void setJoin(final boolean isJoin) { + this.isJoin = isJoin; + } + + public GraphArcDbEntry[] getArcs() { + return arcs; + } + + public void setArcs(final GraphArcDbEntry[] arcs) { + this.arcs = arcs; + } + + public GraphParameterDbEntry[] getParams() { + return params; + } + + public void setParams(final GraphParameterDbEntry[] params) { + this.params = params; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java new file mode 100644 index 00000000..6edcf0f7 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java @@ -0,0 +1,39 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +public class GraphParameterDbEntry extends GraphParameterValueDbEntry implements Serializable { + + private static final long serialVersionUID = 1894419948433994453L; + + private String name; + private List values; + private Map map; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public List getValues() { + return values; + } + + public void setValues(final List values) { + this.values = values; + } + + public Map getMap() { + return map; + } + + public void setMap(final Map map) { + this.map = map; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java new file mode 100644 index 00000000..c04a6e30 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java @@ -0,0 +1,45 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +public class GraphParameterValueDbEntry implements Serializable { + + private static final long serialVersionUID = 7815785723401725707L; + + private String value; + private String ref; + private String property; + private String env; + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + public String getRef() { + return ref; + } + + public void setRef(final String ref) { + this.ref = ref; + } + + public String getProperty() { + return property; + } + + public void setProperty(final String property) { + this.property = property; + } + + public String getEnv() { + return env; + } + + public void setEnv(final String env) { + this.env = env; + } +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java new file mode 100644 index 00000000..dbb0c205 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java @@ -0,0 +1,5 @@ +package eu.dnetlib.manager.wf.model; + +public enum NotificationCondition { + ALWAYS, NEVER, ONLY_SUCCESS, ONLY_FAILED +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java similarity index 81% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java index 0bea598f..4863e27d 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java @@ -1,7 +1,7 @@ -package eu.dnetlib.msro.model.history; +package eu.dnetlib.manager.wf.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Map; import javax.persistence.Column; @@ -32,6 +32,9 @@ public class WfProcessExecution implements Serializable { @Column(name = "process_id") private String processId; + @Column(name = "wf_instance_id") + private String wfInstanceId; + @Column(name = "name") private String name; @@ -43,11 +46,11 @@ public class WfProcessExecution implements Serializable { @Temporal(TemporalType.TIMESTAMP) @Column(name = "start_date") - private Date startDate; + private LocalDateTime startDate; @Temporal(TemporalType.TIMESTAMP) @Column(name = "end_date") - private Date endDate; + private LocalDateTime endDate; @Column(name = "ds_id") private String dsId; @@ -70,6 +73,14 @@ public class WfProcessExecution implements Serializable { this.processId = processId; } + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + public String getName() { return name; } @@ -94,19 +105,19 @@ public class WfProcessExecution implements Serializable { this.status = status; } - public Date getStartDate() { + public LocalDateTime getStartDate() { return startDate; } - public void setStartDate(final Date startDate) { + public void setStartDate(final LocalDateTime startDate) { this.startDate = startDate; } - public Date getEndDate() { + public LocalDateTime getEndDate() { return endDate; } - public void setEndDate(final Date endDate) { + public void setEndDate(final LocalDateTime endDate) { this.endDate = endDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java new file mode 100644 index 00000000..f991bec7 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java @@ -0,0 +1,83 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; +import com.vladmihalcea.hibernate.type.json.JsonStringType; + +@Entity +@Table(name = "workflows") +@TypeDefs({ + @TypeDef(name = "json", typeClass = JsonStringType.class), + @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) +}) +public class WorkflowDbEntry implements Serializable { + + private static final long serialVersionUID = -4684952453322136556L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "name") + private String name; + + @Column(name = "family") + private String family; + + @Column(name = "description") + private String description; + + @Type(type = "jsonb") + @Column(name = "data", columnDefinition = "jsonb") + private GraphNodeDbEntry[] graph; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getFamily() { + return family; + } + + public void setFamily(final String family) { + this.family = family; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public GraphNodeDbEntry[] getGraph() { + return graph; + } + + public void setGraph(final GraphNodeDbEntry[] graph) { + this.graph = graph; + } +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java new file mode 100644 index 00000000..a0ddca0a --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -0,0 +1,195 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; +import com.vladmihalcea.hibernate.type.json.JsonStringType; + +@Entity +@Table(name = "workflow_instances") +@TypeDefs({ + @TypeDef(name = "json", typeClass = JsonStringType.class), + @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) +}) +public class WorkflowInstance implements Serializable { + + private static final long serialVersionUID = 7503841966138333044L; + + @Id + @Column(name = "id") + private String id; + + @Type(type = "jsonb") + @Column(name = "data", columnDefinition = "jsonb") + private Map details = new LinkedHashMap<>(); + + @Column(name = "priority") + private Integer priority; + + @Column(name = "dsid") + private String dsId; + + @Column(name = "dsname") + private String dsName; + + @Column(name = "apiid") + private String apiId; + + @Column(name = "enabled") + private boolean enabled; + + @Column(name = "configured") + private boolean configured; + + @Column(name = "scheduling_enabled") + private boolean schedulingEnabled; + + @Column(name = "scheduling_cron") + private String cronExpression; + + @Column(name = "scheduling_min_interval") + private int cronMinInterval; + + @Column(name = "workflow") + private String workflow; + + @Column(name = "destroy_wf") + private String destroyWf; + + @Column(name = "system_params") + private Map systemParams = new LinkedHashMap<>(); + + @Column(name = "user_params") + private Map userParams = new LinkedHashMap<>(); + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public Map getDetails() { + return details; + } + + public void setDetails(final Map details) { + this.details = details; + } + + public Integer getPriority() { + return priority; + } + + public void setPriority(final Integer priority) { + this.priority = priority; + } + + public String getDsId() { + return dsId; + } + + public void setDsId(final String dsId) { + this.dsId = dsId; + } + + public String getDsName() { + return dsName; + } + + public void setDsName(final String dsName) { + this.dsName = dsName; + } + + public String getApiId() { + return apiId; + } + + public void setApiId(final String apiId) { + this.apiId = apiId; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(final boolean enabled) { + this.enabled = enabled; + } + + public boolean isConfigured() { + return configured; + } + + public void setConfigured(final boolean configured) { + this.configured = configured; + } + + public boolean isSchedulingEnabled() { + return schedulingEnabled; + } + + public void setSchedulingEnabled(final boolean schedulingEnabled) { + this.schedulingEnabled = schedulingEnabled; + } + + public String getCronExpression() { + return cronExpression; + } + + public void setCronExpression(final String cronExpression) { + this.cronExpression = cronExpression; + } + + public int getCronMinInterval() { + return cronMinInterval; + } + + public void setCronMinInterval(final int cronMinInterval) { + this.cronMinInterval = cronMinInterval; + } + + public String getWorkflow() { + return workflow; + } + + public void setWorkflow(final String workflow) { + this.workflow = workflow; + } + + public String getDestroyWf() { + return destroyWf; + } + + public void setDestroyWf(final String destroyWf) { + this.destroyWf = destroyWf; + } + + public Map getSystemParams() { + return systemParams; + } + + public void setSystemParams(final Map systemParams) { + this.systemParams = systemParams; + } + + public Map getUserParams() { + return userParams; + } + + public void setUserParams(final Map userParams) { + this.userParams = userParams; + } +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java new file mode 100644 index 00000000..9e0f177f --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java @@ -0,0 +1,5 @@ +package eu.dnetlib.manager.wf.model; + +public class WorkflowParamDesc { + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java new file mode 100644 index 00000000..9442e279 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java @@ -0,0 +1,68 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; + +@Entity +@Table(name = "workflow_subscriptions") +@IdClass(WorkflowSubscriptionPK.class) +public class WorkflowSubscription implements Serializable { + + private static final long serialVersionUID = -3662770213782581404L; + + @Id + @Column(name = "wf_instance_id") + private String wfInstanceId; + + @Id + @Column(name = "condition") + @Enumerated(EnumType.STRING) + private NotificationCondition condition; + + @Id + @Column(name = "email") + private String email; + + @Column(name = "message_id") + private String messageId; + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + + public NotificationCondition getCondition() { + return condition; + } + + public void setCondition(final NotificationCondition condition) { + this.condition = condition; + } + + public String getEmail() { + return email; + } + + public void setEmail(final String email) { + this.email = email; + } + + public String getMessageId() { + return messageId; + } + + public void setMessageId(final String messageId) { + this.messageId = messageId; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java new file mode 100644 index 00000000..61b3abfc --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java @@ -0,0 +1,39 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +public class WorkflowSubscriptionPK implements Serializable { + + private static final long serialVersionUID = -7569690774071644848L; + + private String wfInstanceId; + + private NotificationCondition condition; + + private String email; + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + + public NotificationCondition getCondition() { + return condition; + } + + public void setCondition(final NotificationCondition condition) { + this.condition = condition; + } + + public String getEmail() { + return email; + } + + public void setEmail(final String email) { + this.email = email; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java index 63691bf7..3cdda237 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java @@ -1,9 +1,11 @@ package eu.dnetlib.utils; import java.io.StringWriter; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Locale; +import java.util.TimeZone; public class DateUtils { @@ -12,7 +14,7 @@ public class DateUtils { private static final long HOUR = MINUTE * 60; private static final long DAY = HOUR * 24; private static final long YEAR = DAY * 365; - private static final SimpleDateFormat ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US); + private static final DateTimeFormatter ISO8601FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault()); public static String elapsedTime(long t) { final StringWriter a = new StringWriter(); @@ -78,7 +80,13 @@ public class DateUtils { } public static String calculate_ISO8601(final long l) { - String result = ISO8601FORMAT.format(new Date(l)); + + final LocalDateTime time = + LocalDateTime.ofInstant(Instant.ofEpochMilli(l), TimeZone + .getDefault() + .toZoneId()); + + String result = time.format(ISO8601FORMAT); // convert YYYYMMDDTHH:mm:ss+HH00 into YYYYMMDDTHH:mm:ss+HH:00 // - note the added colon for the Timezone result = result.substring(0, result.length() - 2) + ":" + result.substring(result.length() - 2); @@ -89,4 +97,8 @@ public class DateUtils { return Long.toString(Math.floorDiv(d, n)); } + public static long now() { + return Instant.now().toEpochMilli(); + } + } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java new file mode 100644 index 00000000..dd1fb957 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java @@ -0,0 +1,10 @@ +package eu.dnetlib.utils; + +public interface Stoppable { + + void stop(); + + void resume(); + + public StoppableDetails getStopDetails(); +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java new file mode 100644 index 00000000..708198c0 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java @@ -0,0 +1,33 @@ +package eu.dnetlib.utils; + +public class StoppableDetails { + + public enum StopStatus { + STOPPED, + STOPPING, + RUNNING + }; + + private final String title; + private final String description; + private final StopStatus status; + + public StoppableDetails(final String title, final String description, final StopStatus status) { + this.title = title; + this.description = description; + this.status = status; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public StopStatus getStatus() { + return status; + } + +} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index a17f71f8..c734c2e3 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -105,16 +105,17 @@ CREATE INDEX ON context_cat_concepts_lvl_2 (parent); -- WF History CREATE TABLE wf_history ( - process_id text PRIMARY KEY, - name text NOT NULL, - family text NOT NULL, - status text NOT NULL, - start_date timestamp NOT NULL, - end_date timestamp NOT NULL, - ds_id text, - ds_name text, - ds_api text, - details jsonb + process_id text PRIMARY KEY, + wf_instance_id text NOT NULL, + name text NOT NULL, + family text NOT NULL, + status text NOT NULL, + start_date timestamp NOT NULL, + end_date timestamp NOT NULL, + ds_id text, + ds_name text, + ds_api text, + details jsonb ); -- Other Resources @@ -241,3 +242,55 @@ GROUP BY md.id, v1.lastupdate, v1.size; +-- Email Templates +CREATE TABLE emails ( + id text PRIMARY KEY, + subject text NOT NULL, + message text NOT NULL +); + +-- Workflows + +CREATE TABLE workflows ( + id text PRIMARY KEY, + name text NOT NULL, + family text NOT NULL, + description text, + data jsonb NOT NULL +); + +CREATE TABLE workflow_instances ( + id text PRIMARY KEY, + details jsonb NOT NULL DEFAULT '{}', + priority int, + dsid text, + dsname text, + apiid text, + enabled boolean NOT NULL DEFAULT false, + configured boolean NOT NULL DEFAULT false, + scheduling_enabled boolean NOT NULL DEFAULT false, + scheduling_cron text, + scheduling_min_interval int, + workflow text REFERENCES workflows(id), + destroy_wf text REFERENCES workflows(id), + system_params jsonb NOT NULL DEFAULT '{}', + user_params jsonb NOT NULL DEFAULT '{}' +); + +CREATE TABLE workflow_expected_params ( + wf_id text REFERENCES workflows(id), + name text, + description text, + type text, + required boolean, + default_value text, + PRIMARY KEY (wf_id, name) +); + +CREATE TABLE workflow_subscriptions ( + wf_instance_id text NOT NULL REFERENCES workflow_instances(id), + condition text NOT NULL, + email text NOT NULL, + message_id text NOT NULL REFERENCES emails(id), + PRIMARY KEY (wf_instance_id, condition, email) +); diff --git a/libs/dnet-is-services/pom.xml b/libs/dnet-is-services/pom.xml index a21a4f9d..709c81b9 100644 --- a/libs/dnet-is-services/pom.xml +++ b/libs/dnet-is-services/pom.xml @@ -24,6 +24,11 @@ jaxen jaxen + + + org.springframework.boot + spring-boot-starter-thymeleaf + diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java index 5f292ce1..f9a1c8a3 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java @@ -9,7 +9,7 @@ import static eu.dnetlib.dsm.utils.DsmMappingUtils.createId; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.sql.Date; +import java.time.LocalDate; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -368,7 +368,7 @@ public class DsmService { private void ensureRegistrationDate(final String dsId) { if (!dsRepository.hasRegistrationdate(dsId)) { log.info("setting registration date for datasource: " + dsId); - dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis())); + dsRepository.setRegistrationDate(dsId, LocalDate.now()); } } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java index 6bf9c977..8e12cb5c 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.Set; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -27,15 +27,15 @@ public class ApiDetails extends ApiIgnoredProperties { private Integer lastCollectionTotal; - private Date lastCollectionDate; + private LocalDateTime lastCollectionDate; private Integer lastAggregationTotal; - private Date lastAggregationDate; + private LocalDateTime lastAggregationDate; private Integer lastDownloadTotal; - private Date lastDownloadDate; + private LocalDateTime lastDownloadDate; private String baseurl; @@ -71,7 +71,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastCollectionTotal; } - public Date getLastCollectionDate() { + public LocalDateTime getLastCollectionDate() { return lastCollectionDate; } @@ -79,7 +79,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastAggregationTotal; } - public Date getLastAggregationDate() { + public LocalDateTime getLastAggregationDate() { return lastAggregationDate; } @@ -87,7 +87,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastDownloadTotal; } - public Date getLastDownloadDate() { + public LocalDateTime getLastDownloadDate() { return lastDownloadDate; } @@ -125,7 +125,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastCollectionDate(final Date lastCollectionDate) { + public ApiDetails setLastCollectionDate(final LocalDateTime lastCollectionDate) { this.lastCollectionDate = lastCollectionDate; return this; } @@ -135,7 +135,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastAggregationDate(final Date lastAggregationDate) { + public ApiDetails setLastAggregationDate(final LocalDateTime lastAggregationDate) { this.lastAggregationDate = lastAggregationDate; return this; } @@ -145,7 +145,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastDownloadDate(final Date lastDownloadDate) { + public ApiDetails setLastDownloadDate(final LocalDateTime lastDownloadDate) { this.lastDownloadDate = lastDownloadDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java index 096a76dd..ef7c0a08 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import java.util.Set; import javax.persistence.Transient; @@ -48,12 +48,12 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { private String languages; - private Date dateofvalidation; + private LocalDate dateofvalidation; @NotBlank private String eoscDatasourceType; - private Date dateofcollection; + private LocalDate dateofcollection; private String platform; @@ -82,9 +82,9 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { private Boolean fullTextDownload; - private Date consentTermsOfUseDate; + private LocalDate consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDate lastConsentTermsOfUseDate; private Set organizations; @@ -95,7 +95,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @Deprecated private String typology; - private Date registrationdate; + private LocalDate registrationdate; public String getId() { return id; @@ -145,7 +145,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return namespaceprefix; } - public Date getDateofvalidation() { + public LocalDate getDateofvalidation() { return dateofvalidation; } @@ -153,7 +153,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return eoscDatasourceType; } - public Date getDateofcollection() { + public LocalDate getDateofcollection() { return dateofcollection; } @@ -277,7 +277,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public DatasourceDetails setDateofvalidation(final Date dateofvalidation) { + public DatasourceDetails setDateofvalidation(final LocalDate dateofvalidation) { this.dateofvalidation = dateofvalidation; return this; } @@ -287,7 +287,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public DatasourceDetails setDateofcollection(final Date dateofcollection) { + public DatasourceDetails setDateofcollection(final LocalDate dateofcollection) { this.dateofcollection = dateofcollection; return this; } @@ -367,11 +367,11 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDate getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceDetails setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -396,20 +396,20 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDate getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceDetails setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } - public Date getRegistrationdate() { + public LocalDate getRegistrationdate() { return registrationdate; } - public DatasourceDetails setRegistrationdate(final Date registrationdate) { + public DatasourceDetails setRegistrationdate(final LocalDate registrationdate) { this.registrationdate = registrationdate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java index e9fe39f8..3451ba70 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import java.util.Set; import javax.validation.constraints.Email; @@ -57,9 +57,9 @@ public class DatasourceDetailsUpdate { private Boolean consentTermsOfUse; - private Date consentTermsOfUseDate; + private LocalDate consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDate lastConsentTermsOfUseDate; private Boolean fullTextDownload; @@ -207,11 +207,11 @@ public class DatasourceDetailsUpdate { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDate getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceDetailsUpdate setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -225,11 +225,11 @@ public class DatasourceDetailsUpdate { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDate getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java index 4a1a3e10..dfea4420 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -13,10 +13,10 @@ public abstract class DatasourceIgnoredProperties { protected String provenanceaction; @JsonIgnore - protected Date releasestartdate; + protected LocalDate releasestartdate; @JsonIgnore - protected Date releaseenddate; + protected LocalDate releaseenddate; @JsonIgnore protected String missionstatementurl; @@ -76,19 +76,19 @@ public abstract class DatasourceIgnoredProperties { this.provenanceaction = provenanceaction; } - public Date getReleasestartdate() { + public LocalDate getReleasestartdate() { return releasestartdate; } - public void setReleasestartdate(final Date releasestartdate) { + public void setReleasestartdate(final LocalDate releasestartdate) { this.releasestartdate = releasestartdate; } - public Date getReleaseenddate() { + public LocalDate getReleaseenddate() { return releaseenddate; } - public void setReleaseenddate(final Date releaseenddate) { + public void setReleaseenddate(final LocalDate releaseenddate) { this.releaseenddate = releaseenddate; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java index b44f19d8..adf99be5 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Set; import javax.validation.constraints.Email; @@ -28,7 +28,7 @@ public class DatasourceSnippetExtended { @Email private String registeredby; - private Date registrationdate; + private LocalDateTime registrationdate; private String eoscDatasourceType; @@ -38,9 +38,9 @@ public class DatasourceSnippetExtended { private Boolean consentTermsOfUse; - private Date consentTermsOfUseDate; + private LocalDateTime consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDateTime lastConsentTermsOfUseDate; private Boolean fullTextDownload; @@ -89,11 +89,11 @@ public class DatasourceSnippetExtended { this.registeredby = registeredby; } - public Date getRegistrationdate() { + public LocalDateTime getRegistrationdate() { return registrationdate; } - public void setRegistrationdate(final Date registrationdate) { + public void setRegistrationdate(final LocalDateTime registrationdate) { this.registrationdate = registrationdate; } @@ -118,11 +118,11 @@ public class DatasourceSnippetExtended { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDateTime getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceSnippetExtended setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -170,11 +170,11 @@ public class DatasourceSnippetExtended { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDateTime getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java index 2a87622f..be9b507a 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -14,7 +14,7 @@ public class OrganizationIgnoredProperties { protected String collectedfrom; @JsonIgnore - protected Date dateofcollection; + protected LocalDateTime dateofcollection; @JsonIgnore protected String provenanceaction; @@ -38,11 +38,11 @@ public class OrganizationIgnoredProperties { this.collectedfrom = collectedfrom; } - public Date getDateofcollection() { + public LocalDateTime getDateofcollection() { return dateofcollection; } - public void setDateofcollection(final Date dateofcollection) { + public void setDateofcollection(final LocalDateTime dateofcollection) { this.dateofcollection = dateofcollection; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java index b7319c14..c426dee0 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.repository; -import java.sql.Date; +import java.time.LocalDate; import java.util.Optional; import javax.transaction.Transactional; @@ -64,7 +64,7 @@ public interface DatasourceRepository extends JpaRepository, @Modifying @Transactional @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1") - void setRegistrationDate(String id, Date registrationdate); + void setRegistrationDate(String id, LocalDate registrationdate); @Modifying @Transactional diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java index efe75a43..666842ce 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.utils; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -98,7 +98,7 @@ public class DsmMappingUtils { final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); o.setId(prefix + ID_SEPARATOR + o.getLegalname()); if (o.getDateofcollection() == null) { - o.setDateofcollection(new Date(System.currentTimeMillis())); + o.setDateofcollection(LocalDateTime.now()); } o.setCollectedfrom(dbe.getCollectedfrom()); }); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java index 10da2b98..42fbb7cb 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java @@ -1,6 +1,6 @@ package eu.dnetlib.is.resource; -import java.util.Date; +import java.time.LocalDateTime; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -65,7 +65,7 @@ public class SimpleResourceService { resourceValidator.validate(type, content); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); final SimpleResource res = new SimpleResource(); res.setId(UUID.randomUUID().toString()); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java new file mode 100644 index 00000000..31ae50cd --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java @@ -0,0 +1,51 @@ +package eu.dnetlib.manager.wf; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.TimeZone; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.wf.repository.WfProcessExecutionRepository; + +@Service +public class WorkflowLogger { + + @Autowired + private WfProcessExecutionRepository wfProcessExecutionRepository; + + public List history(final int total, final Long from, final Long to) { + if (from == null && to == null) { + return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList(); + } else { + + final LocalDateTime fromTime = from != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(from), TimeZone + .getDefault() + .toZoneId()) : LocalDateTime.MIN; + final LocalDateTime toTime = to != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(to), TimeZone + .getDefault() + .toZoneId()) : LocalDateTime.MAX; + + return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(fromTime, toTime); + } + } + + public WfProcessExecution getProcessExecution(final String processId) { + return wfProcessExecutionRepository.findById(processId).get(); + } + + public void saveProcessExecution(final WfProcessExecution proc) { + wfProcessExecutionRepository.save(proc); + } + + public Optional getLastExecutionForInstance(final String id) { + return wfProcessExecutionRepository.findOneByWfInstanceIdOrderByEndDateAsc(id); + } + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java new file mode 100644 index 00000000..5e98cf23 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.repository; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.EmailTemplate; + +public interface EmailTemplateRepository extends JpaRepository { + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java new file mode 100644 index 00000000..cf58142d --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java @@ -0,0 +1,16 @@ +package eu.dnetlib.manager.wf.repository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.WfProcessExecution; + +public interface WfProcessExecutionRepository extends JpaRepository { + + List findByEndDateBetweenOrderByEndDateDesc(LocalDateTime start, LocalDateTime end); + + Optional findOneByWfInstanceIdOrderByEndDateAsc(String id); +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java new file mode 100644 index 00000000..9e007e54 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.repository; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.WorkflowDbEntry; + +public interface WorkflowDbEntryRepository extends JpaRepository { + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java new file mode 100644 index 00000000..cb477905 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.repository; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.WorkflowInstance; + +public interface WorkflowInstanceRepository extends JpaRepository { + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java new file mode 100644 index 00000000..ff493ca7 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java @@ -0,0 +1,13 @@ +package eu.dnetlib.manager.wf.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.WorkflowSubscription; +import eu.dnetlib.manager.wf.model.WorkflowSubscriptionPK; + +public interface WorkflowSubscriptionRepository extends JpaRepository { + + List findByWfInstanceId(String wfInstanceId); +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java deleted file mode 100644 index be3a8c9a..00000000 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.dnetlib.msro.history.repository; - -import java.util.Date; -import java.util.List; - -import org.springframework.data.jpa.repository.JpaRepository; - -import eu.dnetlib.msro.model.history.WfProcessExecution; - -public interface WfProcessExecutionRepository extends JpaRepository { - - List findByEndDateBetweenOrderByEndDateDesc(Date start, Date end); -} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java new file mode 100644 index 00000000..c5396b69 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java @@ -0,0 +1,147 @@ +package eu.dnetlib.notifications.mail; + +import java.util.Date; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import javax.annotation.PostConstruct; +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.common.base.Splitter; + +import eu.dnetlib.manager.wf.repository.EmailTemplateRepository; + +@Service +public class EmailDispatcher { + + private static final Log log = LogFactory.getLog(EmailDispatcher.class); + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + + @Value("${'dnet.configuration.mail.sender.email'}") + private String from; + @Value("${'dnet.configuration.mail.sender.name'}") + private String fromName; + @Value("${'dnet.configuration.mail.cc'}") + private String cc; + @Value("${'dnet.configuration.mail.smtp.host'}") + private String smtpHost; + @Value("${'dnet.configuration.mail.smtp.port'}") + private final int smtpPort = 587; + @Value("${'dnet.configuration.mail.smtp.user'}") + private String smtpUser; + @Value("${'dnet.configuration.mail.smtp.password'}") + private String smtpPassword; + @Value("${'dnet.configuration.baseUrl'}") + private String baseUrl; + @Value("${'dnet.configuration.infrastructure'}") + private String infrastructure; + + @Autowired + private EmailTemplateRepository emailTemplateRepository; + + @PostConstruct + private void init() { + new Thread(() -> { + while (true) { + try { + final Message message = this.queue.take(); + if (message != null) { + try { + log.info("Sending mail..."); + Transport.send(message); + log.info("...sent"); + } catch (final MessagingException e) { + log.error("Error sending email", e); + this.queue.add(message); + } + } + } catch (final InterruptedException e1) { + throw new RuntimeException(e1); + } + } + }).run(); + } + + public void sendMail(final String to, final String subject, final String message) { + try { + final Session session = Session.getInstance(obtainProperties(), obtainAuthenticator()); + + final MimeMessage mimeMessage = new MimeMessage(session); + mimeMessage.setFrom(new InternetAddress(this.from, this.fromName)); + mimeMessage.setSubject(subject); + mimeMessage.setContent(message, "text/html; charset=utf-8"); + mimeMessage.setSentDate(new Date()); + + mimeMessage.addRecipient(Message.RecipientType.TO, new InternetAddress(to)); + + if (this.cc != null && !this.cc.isEmpty()) { + for (final String aCC : Splitter.on(",").omitEmptyStrings().trimResults().split(cc)) { + mimeMessage.addRecipient(Message.RecipientType.CC, new InternetAddress(aCC)); + } + } + + this.queue.add(mimeMessage); + + log.info("Mail to " + to + " in queue"); + } catch (final Exception e) { + log.error("Error sending mail", e); + } + } + + public void sendStoredMail(final String to, final String emailId, final Map params) { + + // TODO use a real template library + emailTemplateRepository.findById(emailId).ifPresent(tmpl -> { + String msg = tmpl.getMessage(); + String subject = tmpl.getSubject(); + + for (final Entry e : params.entrySet()) { + msg = msg.replaceAll("{" + e.getKey() + "}", e.getValue().toString()); + subject = subject.replaceAll("{" + e.getKey() + "}", e.getValue().toString()); + } + sendMail(to, subject, msg); + }); + + } + + private Properties obtainProperties() { + final Properties props = new Properties(); + props.put("mail.transport.protocol", "smtp"); + props.put("mail.smtp.host", this.smtpHost); + props.put("mail.smtp.port", this.smtpPort); + props.put("mail.smtp.auth", Boolean.toString(this.smtpUser != null && !this.smtpUser.isEmpty())); + return props; + } + + private Authenticator obtainAuthenticator() { + if (this.smtpUser == null || this.smtpUser.isEmpty()) { return null; } + + return new Authenticator() { + + private final PasswordAuthentication authentication = new PasswordAuthentication(EmailDispatcher.this.smtpUser, EmailDispatcher.this.smtpPassword); + + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return this.authentication; + } + + }; + } + +} diff --git a/libs/dnet-wf-service/pom.xml b/libs/dnet-wf-service/pom.xml new file mode 100644 index 00000000..6a398249 --- /dev/null +++ b/libs/dnet-wf-service/pom.xml @@ -0,0 +1,48 @@ + + + + eu.dnetlib.dhp + libs + 3.3.3-SNAPSHOT + ../ + + + 4.0.0 + dnet-wf-service + jar + + + + eu.dnetlib.dhp + dnet-is-services + ${project.version} + + + + eu.dnetlib.dhp + dnet-data-services + ${project.version} + + + + + org.junit.jupiter + junit-jupiter + test + + + + org.mockito + mockito-core + test + + + + org.mockito + mockito-junit-jupiter + test + + + + + diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java new file mode 100644 index 00000000..5301a8b0 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java @@ -0,0 +1,26 @@ +package eu.dnetlib.manager.wf; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class NodeInfo { + + private String name; + private Map params = new LinkedHashMap<>(); + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public Map getParams() { + return params; + } + + public void setParams(final Map params) { + this.params = params; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java new file mode 100644 index 00000000..8955711d --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java @@ -0,0 +1,20 @@ +package eu.dnetlib.manager.wf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Service; + +@Service +public class WfManager { + + @Autowired + private ApplicationContext applicationContext; + + /* + * public Object obtainBaseNode(final String name) { return applicationContext.getBean(name); } + * + * public NodeInfo obtainInfo(final BaseNode node) { final NodeInfo info = new NodeInfo(); final WfNode annotation = + * node.getClass().getAnnotation(WfNode.class); info.setName(annotation.name()); for (final WfParam p : annotation.inputParams()) { + * info.getParams().put(p.name(), p.type()); } return info; } + */ +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java new file mode 100644 index 00000000..ee5f811f --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.annotations; + +public enum StreamMimeType { + XML, + JSON, + TEXT, + UNDEFINED +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java new file mode 100644 index 00000000..fa5930f1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java @@ -0,0 +1,20 @@ +package eu.dnetlib.manager.wf.annotations; + +public @interface WfNode { + + String name(); + + WfNodeOperation operation(); + + Class inputStreamType() default void.class; + + StreamMimeType inputStreamMimeType() default StreamMimeType.UNDEFINED; + + Class outputStreamType() default void.class; + + StreamMimeType outputStreamMimeType() default StreamMimeType.UNDEFINED; + + WfParam[] inputParams() default {}; + + WfParam[] outputParams() default {}; +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java new file mode 100644 index 00000000..05a88378 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java @@ -0,0 +1,11 @@ +package eu.dnetlib.manager.wf.annotations; + +public enum WfNodeOperation { + CREATE, + DROP, + READ, + WRITE, + PRODUCER, + TRANSFORM, + SETENV +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java new file mode 100644 index 00000000..5566dc12 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.annotations; + +public @interface WfParam { + + String name(); + + String type(); +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java new file mode 100644 index 00000000..4f07469c --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -0,0 +1,138 @@ +package eu.dnetlib.manager.wf.cron; + +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.support.CronExpression; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.wf.WorkflowLogger; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.utils.DateUtils; + +@Service +public class ScheduledWorkflowLauncher { + + private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class); + + private static final DateUtils dateUtils = new DateUtils(); + + @Autowired + private WorkflowExecutor workflowExecutor; + + @Autowired + private ProcessRegistry processRegistry; + + @Autowired + private WorkflowInstanceRepository workflowInstanceRepository; + + @Autowired + private WorkflowLogger logger; + + @Value("${'dnet.workflow.scheduler.windowSize'}") + private int windowSize; // 1800000 are 30 minutes + + @Scheduled(cron = "${'dnet.workflow.scheduler.cron'}") + public void verifySheduledWorkflows() { + log.debug("Verifying scheduled workflows - START"); + + workflowInstanceRepository.findAll() + .stream() + .filter(WorkflowInstance::isEnabled) + .filter(WorkflowInstance::isConfigured) + .filter(WorkflowInstance::isSchedulingEnabled) + .filter(this::isNotRunning) + .filter(this::isReady) + .forEach(instance -> { + try { + workflowExecutor.startWorkflowInstance(instance, null, null); + } catch (final Exception e) { + log.error("Error launching scheduled wf instance: " + instance.getId(), e); + } + }); + log.debug("Verifying scheduled workflows - END"); + } + + private boolean isReady(final WorkflowInstance instance) { + final LocalDateTime lastExecutionDate = calculateLastExecutionDate(instance.getId()); + + final LocalDateTime now = LocalDateTime.now(); + + final String cron = instance.getCronExpression(); + + if (CronExpression.isValidExpression(cron)) { + final int minInterval = instance.getCronMinInterval(); // in minutes + + final boolean res; + if (lastExecutionDate != null) { + final long elapsed = ChronoUnit.MINUTES.between(lastExecutionDate, now); + res = elapsed > minInterval && verifyCron(cron, now); + } else { + res = verifyCron(cron, now); + } + + if (log.isDebugEnabled()) { + log.debug("**************************************************************"); + log.debug("WORKFLOW INSTANCE ID : " + instance.getId()); + log.debug("NOW : " + now); + log.debug("LAST EXECUTION DATE : " + lastExecutionDate); + log.debug("MIN INTERVAL (minutes) : " + minInterval); + log.debug("WINDOW SIZE (ms) : " + windowSize); + log.debug("MUST BE EXECUTED : " + res); + log.debug("**************************************************************"); + } + + return res; + } + + return false; + } + + private LocalDateTime calculateLastExecutionDate(final String id) { + return logger.getLastExecutionForInstance(id) + .map(e -> e.getEndDate()) + .orElse(LocalDateTime.MIN); + } + + private boolean verifyCron(final String cronExpression, final LocalDateTime now) { + try { + final CronExpression cron = CronExpression.parse(cronExpression); + + final LocalDateTime date = now.minus(windowSize, ChronoUnit.MINUTES); + final LocalDateTime nextDate = cron.next(date); + + if (log.isDebugEnabled()) { + log.debug("NEXT EXECUTION DATE: " + nextDate); + log.debug("FIRED : " + nextDate.isBefore(now)); + } + return nextDate.isBefore(now); + } catch (final Exception e) { + log.error("Error calculating next cron event: " + cronExpression, e); + return false; + } + } + + private boolean isNotRunning(final WorkflowInstance instance) { + for (final WorkflowProcess p : processRegistry.findProcsByOtherId(instance.getId())) { + switch (p.getStatus()) { + case CREATED: + return false; + case EXECUTING: + return false; + default: + break; + } + } + return true; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java new file mode 100644 index 00000000..3e8807f3 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java @@ -0,0 +1,45 @@ +package eu.dnetlib.manager.wf.nodes; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import eu.dnetlib.manager.wf.workflows.procs.Env; +import eu.dnetlib.manager.wf.workflows.procs.Token; + +public abstract class AbstractJobNode extends ProcessNode { + + private static final Log log = LogFactory.getLog(SimpleJobNode.class); + + protected void doExecute(final Token token) { + try { + log.debug("START NODE: " + getBeanName()); + beforeStart(token); + final String arc = execute(token.getEnv()); + beforeCompleted(token); + log.debug("END NODE (SUCCESS): " + getBeanName()); + + token.release(arc); + + } catch (final Throwable e) { + log.error("got exception while executing workflow node", e); + log.debug("END NODE (FAILED): " + getBeanName()); + beforeFailed(token); + token.releaseAsFailed(e); + } + } + + abstract protected String execute(final Env env) throws Exception; + + protected void beforeStart(final Token token) { + // For optional overwrites + } + + protected void beforeCompleted(final Token token) { + // For optional overwrites + } + + protected void beforeFailed(final Token token) { + // For optional overwrites + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java new file mode 100644 index 00000000..fe84b46d --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java @@ -0,0 +1,28 @@ +package eu.dnetlib.manager.wf.nodes; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +public abstract class AsyncJobNode extends AbstractJobNode { + + /** + * logger. + */ + private static final Log log = LogFactory.getLog(AsyncJobNode.class); + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Override + public final void execute(final Token token) { + + log.info("executing async node"); + + executor.execute(() -> doExecute(token)); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java new file mode 100644 index 00000000..b8321c00 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java @@ -0,0 +1,29 @@ +package eu.dnetlib.manager.wf.nodes; + +import java.util.function.Supplier; +import java.util.stream.Stream; + +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import eu.dnetlib.manager.wf.annotations.StreamMimeType; +import eu.dnetlib.manager.wf.annotations.WfNode; +import eu.dnetlib.manager.wf.annotations.WfNodeOperation; +import eu.dnetlib.manager.wf.annotations.WfParam; + +@Component("oai_collect") +@Scope("prototype") +@WfNode(name = "oai_collect", operation = WfNodeOperation.PRODUCER, inputParams = { + @WfParam(name = "configuration", type = "ApiDescriptor.class"), +}, outputStreamType = String.class, outputStreamMimeType = StreamMimeType.XML) +public class CollectOAINode implements Supplier> { + + private String datasourceID; + + @Override + public Stream get() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java new file mode 100644 index 00000000..32a8c750 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java @@ -0,0 +1,21 @@ +package eu.dnetlib.manager.wf.nodes; + +import eu.dnetlib.manager.wf.workflows.graph.Arc; +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 26/11/15. + */ +public final class DefaultJobNode extends ProcessNode { + + public DefaultJobNode(final String name) { + super(); + setNodeName(name); + } + + @Override + public void execute(final Token token) { + token.setNextArc(Arc.DEFAULT_ARC); + token.release(); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java new file mode 100644 index 00000000..1818bda1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -0,0 +1,78 @@ +package eu.dnetlib.manager.wf.nodes; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import eu.dnetlib.manager.wf.workflows.graph.Arc; +import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; +import eu.dnetlib.manager.wf.workflows.util.SubWorkflowProgressProvider; + +/** + * Created by michele on 18/11/15. + */ +public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { + + private static final Log log = LogFactory.getLog(LaunchWorkflowJobNode.class); + + private String wfInstanceId; + + @Autowired + private WorkflowExecutor executor; + + @Autowired + private ProcessRegistry processRegistry; + + private WorkflowProcess process; + + @Override + public final void execute(final Token token) { + + try { + final String procId = executor.startWorkflowInstance(getWfInstanceId(), new ProcessCallback() { + + @Override + public void onSuccess() { + log.debug("Child workflow has been completed successfully"); + token.setNextArc(Arc.DEFAULT_ARC); + token.release(); + } + + @Override + public void onFail() { + log.error("Child workflow is failed"); + token.releaseAsFailed("Child workflow is failed"); + } + }, process.getWfInstanceId()); + + if (log.isDebugEnabled()) { + log.debug("The child workflow [instance: " + getWfInstanceId() + "] is starting with procId: " + procId); + } + + token.setProgressProvider(new SubWorkflowProgressProvider(procId, processRegistry)); + + } catch (final Throwable e) { + log.error("got exception while launching child workflow", e); + token.releaseAsFailed(e); + } + } + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + + @Override + public void setProcess(final WorkflowProcess process) { + this.process = process; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java new file mode 100644 index 00000000..ea9aa2cb --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.nodes; + +public enum NodeStatus { + CONFIGURED, + NOT_CONFIGURED, + DISABLED, + SYSTEM +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java new file mode 100644 index 00000000..5052bb31 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java @@ -0,0 +1,39 @@ +package eu.dnetlib.manager.wf.nodes; + +import org.springframework.beans.factory.BeanNameAware; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 19/11/15. + */ +public abstract class ProcessNode implements BeanNameAware { + + private String beanName; + + private String nodeName; + + public abstract void execute(final Token token); + + public String getBeanName() { + return this.beanName; + } + + @Override + public void setBeanName(final String beanName) { + this.beanName = beanName; + } + + public String getNodeName() { + return this.nodeName; + } + + public void setNodeName(final String nodeName) { + this.nodeName = nodeName; + } + + @Override + public String toString() { + return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java new file mode 100644 index 00000000..1dd14429 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java @@ -0,0 +1,12 @@ +package eu.dnetlib.manager.wf.nodes; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +public abstract class SimpleJobNode extends AbstractJobNode { + + @Override + public final void execute(final Token token) { + doExecute(token); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java new file mode 100644 index 00000000..022d3cec --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java @@ -0,0 +1,17 @@ +package eu.dnetlib.manager.wf.nodes; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 26/11/15. + */ +public class SuccessNode extends ProcessNode { + + public SuccessNode() { + super(); + setNodeName("success"); + } + + @Override + public void execute(final Token token) {} +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java new file mode 100644 index 00000000..e844d6d1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java @@ -0,0 +1,46 @@ +package eu.dnetlib.manager.wf.notification; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.wf.model.NotificationCondition; +import eu.dnetlib.manager.wf.repository.WorkflowSubscriptionRepository; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status; +import eu.dnetlib.notifications.mail.EmailDispatcher; + +@Service +public class EmailSender { + + private static final Log log = LogFactory.getLog(EmailSender.class); + + @Autowired + private WorkflowSubscriptionRepository wfSubscriptionRepository; + + @Autowired + private EmailDispatcher dispatcher; + + public void sendMails(final WorkflowProcess proc) { + + wfSubscriptionRepository.findByWfInstanceId(proc.getWfInstanceId()).forEach(s -> { + if (s.getCondition() == NotificationCondition.ALWAYS || + s.getCondition() == NotificationCondition.ONLY_FAILED && proc.getStatus() == Status.FAILURE || + s.getCondition() == NotificationCondition.ONLY_SUCCESS && proc.getStatus() == Status.SUCCESS) { + try { + final Map params = new HashMap<>(); + + dispatcher.sendStoredMail(s.getEmail(), s.getMessageId(), params); + + } catch (final Exception e) { + log.error("Error sending mail to " + s.getEmail(), e); + } + } + + }); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java new file mode 100644 index 00000000..a0347f36 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java @@ -0,0 +1,35 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import org.apache.commons.lang3.StringUtils; + +public class Arc { + + public static final String DEFAULT_ARC = null; + + private final String name; + private final String from; + private final String to; + + public Arc(final String name, final String from, final String to) { + this.name = name; + this.from = from; + this.to = to; + } + + public String getName() { + return this.name; + } + + public String getFrom() { + return this.from; + } + + public String getTo() { + return this.to; + } + + @Override + public String toString() { + return String.format("[ %s: %s -> %s ]", StringUtils.isBlank(this.name) ? "DEFAULT" : this.name, this.from, this.to); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java new file mode 100644 index 00000000..c9027e7f --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java @@ -0,0 +1,80 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; + +public class Graph { + + private final Map nodes = new HashMap<>(); + private List arcs = new ArrayList<>(); + + public void addArc(final Arc arc) { + this.arcs.add(arc); + } + + public void addNode(final GraphNode node) { + this.nodes.put(node.getName(), node); + } + + public Set nodeNames() { + return this.nodes.keySet(); + } + + public Collection nodes() { + return this.nodes.values(); + } + + public GraphNode getNode(final String name) { + return this.nodes.get(name); + } + + public List getArcs() { + return this.arcs; + } + + public void setArcs(final List arcs) { + this.arcs = arcs; + } + + public Set startNodes() { + final Set res = new HashSet<>(); + for (final GraphNode n : this.nodes.values()) { + if (n.isStart()) { + res.add(n); + } + } + return res; + } + + public Set nextNodes(final GraphNode current, final String arcName) { + final Set res = new HashSet<>(); + for (final Arc arc : this.arcs) { + if (StringUtils.equals(arc.getFrom(), current.getName()) && StringUtils.equals(arc.getName(), arcName)) { + res.add(this.nodes.get(arc.getTo())); + } + } + return res; + } + + public int getNumberOfIncomingArcs(final GraphNode node) { + int count = 0; + for (final Arc arc : this.arcs) { + if (arc.getTo().equals(node.getName())) { + count++; + } + } + return count; + } + + @Override + public String toString() { + return "\n************************\nNodes: " + this.nodes + "\nArcs: " + this.arcs + "\n************************\n"; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java new file mode 100644 index 00000000..18f5d2d6 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -0,0 +1,180 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import com.google.common.collect.Sets; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.GraphArcDbEntry; +import eu.dnetlib.manager.wf.model.GraphNodeDbEntry; +import eu.dnetlib.manager.wf.model.GraphParameterDbEntry; +import eu.dnetlib.manager.wf.model.GraphParameterValueDbEntry; +import eu.dnetlib.manager.wf.workflows.util.NodeHelper; + +@Service +public class GraphLoader { + + private static final Log log = LogFactory.getLog(GraphLoader.class); + + private final String regExRef = "\\$\\{(\\w*)\\}"; + + final Pattern pattern = Pattern.compile(regExRef, Pattern.MULTILINE); + + @Autowired + private NodeHelper nodeHelper; + + @Autowired + private Environment env; + + public Graph loadGraph(final GraphNodeDbEntry[] workflowGraph, final Map globalParams) throws WorkflowManagerException { + final Graph graph = new Graph(); + + for (final GraphNodeDbEntry node : workflowGraph) { + final String nodeName = node.getName(); + final String nodeType = node.getType(); + final boolean isStart = node.isStart(); + final boolean isJoin = node.isJoin(); + + final Map params = calculateParamsForNode(node, globalParams); + + if (isStart) { + graph.addNode(GraphNode.newStartNode(nodeName, nodeType, params)); + } else if (isJoin) { + graph.addNode(GraphNode.newJoinNode(nodeName, nodeType, params)); + } else { + graph.addNode(GraphNode.newNode(nodeName, nodeType, params)); + } + + if (graph.getArcs() != null) { + for (final GraphArcDbEntry a : node.getArcs()) { + final String arcName = a.getName(); + final String to = a.getTo(); + graph.addArc(new Arc(StringUtils.isNotBlank(arcName) ? arcName : Arc.DEFAULT_ARC, nodeName, to)); + } + } + + graph.addNode(GraphNode.newSuccessNode()); + } + + checkValidity(graph); + + return graph; + } + + public Map calculateParamsForNode(final GraphNodeDbEntry node, final Map globalParams) { + + final Map params = new HashMap<>(); + + if (node.getParams() != null) { + for (final GraphParameterDbEntry p : node.getParams()) { + + final String pName = p.getName(); + + final GraphNodeParameter pValue = calculateSimpleValue(p, globalParams); + + if (pValue != null) { + params.put(pName, pValue); + } else if (p.getMap() != null) { + + final Map map = p.getMap() + .entrySet() + .stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> { + final GraphNodeParameter gnp = calculateSimpleValue(e.getValue(), globalParams); + if (gnp == null) { + final String msg = String.format("missing value for param: \"%s\"", e.getKey()); + log.debug(msg); + return GraphNodeParameter.newNullParam(); + } + return gnp; + })); + params.put(pName, GraphNodeParameter.newMapParam(map)); + } else if (p.getValues() != null) { + final List list = p.getValues() + .stream() + .map(e -> calculateSimpleValue(e, globalParams)) + .collect(Collectors.toList()); + params.put(pName, GraphNodeParameter.newListParam(list)); + } + } + } + + return params; + } + + private GraphNodeParameter calculateSimpleValue(final GraphParameterValueDbEntry graphValue, final Map globalParams) { + String value = graphValue.getValue(); + final String ref = graphValue.getRef(); + final String prop = graphValue.getProperty(); + final String envRef = graphValue.getEnv(); + + if (StringUtils.isNotBlank(ref) && StringUtils.isNotBlank(globalParams.get(ref))) { + return GraphNodeParameter.newSimpleParam(globalParams.get(ref)); + } else if (StringUtils.isNotBlank(envRef)) { + return GraphNodeParameter.newEnvParam(envRef); + } else if (StringUtils.isNotBlank(value)) { + final Matcher matcher = pattern.matcher(value); + while (matcher.find()) { + final String rName = matcher.group(1); + final String rValue = globalParams.get(rName); + if (StringUtils.isBlank(rValue)) { return null; } + value = value.replaceAll(Pattern.quote(matcher.group(0)), rValue); + System.out.println("NEW VALUE " + value); + } + return GraphNodeParameter.newSimpleParam(value); + } else if (StringUtils.isNotBlank(prop)) { + return GraphNodeParameter.newSimpleParam(env.getProperty(prop)); + } else { + return null; + } + + } + + private void checkValidity(final Graph graph) throws WorkflowManagerException { + + final Set nodesFromArcs = new HashSet<>(); + + boolean foundSuccess = false; + boolean foundStart = false; + + for (final Arc arc : graph.getArcs()) { + if (StringUtils.isBlank(arc.getFrom()) || StringUtils.isBlank(arc.getFrom())) { + throw new WorkflowManagerException("Invalid arc: missing from e/o to"); + } + if (StringUtils.equals(arc.getTo(), GraphNode.SUCCESS_NODE)) { + foundSuccess = true; + } + nodesFromArcs.add(arc.getFrom()); + nodesFromArcs.add(arc.getTo()); + } + + if (!foundSuccess) { throw new WorkflowManagerException("Arc to success not found"); } + + final Set diff = Sets.symmetricDifference(graph.nodeNames(), nodesFromArcs); + if (!diff.isEmpty()) { throw new WorkflowManagerException("Missing or invalid nodes in arcs: " + diff); } + + for (final GraphNode n : graph.nodes()) { + if (StringUtils.isBlank(n.getName())) { throw new WorkflowManagerException("Invalid node: missing name"); } + if (n.isStart()) { + foundStart = true; + } + if (!this.nodeHelper.isValidType(n.getType())) { throw new WorkflowManagerException("Invalid node type: " + n.getType()); } + } + if (!foundStart) { throw new WorkflowManagerException("Start node not found"); } + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java new file mode 100644 index 00000000..db9c5c67 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java @@ -0,0 +1,149 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.io.StringWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class GraphNode { + + public static final String SUCCESS_NODE = "success"; + + private final String name; + private final String type; + private final boolean isStart; + private final boolean isJoin; + private final boolean isSucessNode; + private final Map nodeParams; + + private GraphNode(final String name, + final String type, + final boolean isStart, + final boolean isJoin, + final boolean isSuccessNode, + final Map nodeParams) { + this.name = name; + this.type = type; + this.isStart = isStart; + this.isJoin = isJoin; + this.isSucessNode = isSuccessNode; + this.nodeParams = nodeParams; + } + + public static GraphNode newNode(final String name, + final String type, + final Map nodeParams) { + return new GraphNode(name, type, false, false, false, nodeParams); + } + + public static GraphNode newStartNode(final String name, + final String type, + final Map nodeParams) { + return new GraphNode(name, type, true, false, false, nodeParams); + } + + public static GraphNode newJoinNode(final String name, + final String type, + final Map nodeParams) { + return new GraphNode(name, type, false, true, false, nodeParams); + } + + public static GraphNode newSuccessNode() { + return new GraphNode(SUCCESS_NODE, null, false, true, true, null); + } + + public String getName() { + return this.name; + } + + public String getType() { + return this.type; + } + + public boolean isStart() { + return this.isStart; + } + + public boolean isJoin() { + return this.isJoin; + } + + public boolean isSucessNode() { + return this.isSucessNode; + } + + @Override + public String toString() { + final StringWriter sw = new StringWriter(); + sw.append("[ name: "); + sw.append(this.name); + if (StringUtils.isNotBlank(this.type)) { + sw.append(", type: "); + sw.append(this.type); + } + if (isStart()) { + sw.append(" - isStart"); + } + if (isJoin()) { + sw.append(" - isJoin"); + } + sw.append(" ]"); + return sw.toString(); + } + + public Map getNodeParams() { + return this.nodeParams; + } + + public Map resolveParamsWithNoEnv() { + return resolveParams(null); + } + + @SuppressWarnings("unchecked") + public Map resolveParams(final Env env) { + final Map params = new HashMap<>(); + + if (this.nodeParams != null) { + + for (final Map.Entry e : this.nodeParams.entrySet()) { + final String pName = e.getKey(); + final GraphNodeParameter param = e.getValue(); + + if (param.isEnvParam()) { + params.put(pName, resolveFromEnv(param, env)); + } else if (param.isMap()) { + + final Map map = new HashMap<>(); + + for (final Map.Entry e1 : ((Map) param.getValue()).entrySet()) { + map.put(e1.getKey(), e1.getValue().isEnvParam() ? resolveFromEnv(e1.getValue(), env) : e1.getValue().getValue()); + } + + params.put(pName, map); + + } else if (param.isList()) { + params.put(pName, ((List) param.getValue()) + .stream() + .map(p -> p.isEnvParam() ? resolveFromEnv(p, env) : p.getValue()) + .collect(Collectors.toList())); + + } else { + params.put(pName, param.getValue()); + } + + } + } + + return params; + } + + private Object resolveFromEnv(final GraphNodeParameter param, final Env env) { + return env != null ? env.getAttribute(param.getEnvVariable()) : "[this value will be resolved using the runtime ENV]"; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java new file mode 100644 index 00000000..bbe054eb --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java @@ -0,0 +1,56 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.util.List; +import java.util.Map; + +public class GraphNodeParameter { + + private final Object value; + private final String envVariable; + + private GraphNodeParameter(final Object value, final String envVariable) { + this.value = value; + this.envVariable = envVariable; + } + + public static GraphNodeParameter newNullParam() { + return new GraphNodeParameter(null, null); + } + + public static GraphNodeParameter newSimpleParam(final Object value) { + return new GraphNodeParameter(value, null); + } + + public static GraphNodeParameter newMapParam(final Map map) { + return new GraphNodeParameter(map, null); + } + + public static GraphNodeParameter newListParam(final List list) { + return new GraphNodeParameter(list, null); + } + + public static GraphNodeParameter newEnvParam(final String envVariable) { + return new GraphNodeParameter(null, envVariable); + } + + public Object getValue() { + return this.value; + } + + public boolean isEnvParam() { + return this.envVariable != null; + } + + public String getEnvVariable() { + return this.envVariable; + } + + public boolean isMap() { + return this.value != null && (this.value instanceof Map); + } + + public boolean isList() { + return this.value != null && (this.value instanceof List); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java new file mode 100644 index 00000000..30ef34e7 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java @@ -0,0 +1,54 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by michele on 23/11/15. + */ +public class Env { + + private final Map attrs; + + public Env() { + this.attrs = new HashMap<>(); + } + + public Env(final Map attrs) { + this.attrs = attrs; + } + + public Map getAttributes() { + return attrs; + } + + public void clear() { + attrs.clear(); + } + + public void addAttributes(final Map map) { + if (map != null) { + attrs.putAll(map); + } + } + + public void setAttribute(final String name, final Object value) { + attrs.put(name, value); + } + + public Object getAttribute(final String name) { + return attrs.get(name); + } + + public T getAttribute(final String name, Class clazz) { + return clazz.cast(attrs.get(name)); + } + + public boolean hasAttribute(final String name) { + return attrs.containsKey(name); + } + + public Object removeAttribute(final String name) { + return attrs.remove(name); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java new file mode 100644 index 00000000..37645ccb --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java @@ -0,0 +1,10 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +/** + * Created by michele on 24/11/15. + */ +public interface ProcessAware { + + void setProcess(WorkflowProcess process); + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java new file mode 100644 index 00000000..64beaccd --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -0,0 +1,203 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.common.base.Throwables; + +import eu.dnetlib.manager.wf.WorkflowLogger; +import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.wf.nodes.ProcessNode; +import eu.dnetlib.manager.wf.notification.EmailSender; +import eu.dnetlib.manager.wf.workflows.graph.GraphNode; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status; +import eu.dnetlib.manager.wf.workflows.util.NodeHelper; +import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; + +@Service +public class ProcessEngine { + + private static final Log log = LogFactory.getLog(ProcessEngine.class); + + @Autowired + private NodeHelper nodeHelper; + @Autowired + private EmailSender emailSender; + @Autowired + private WorkflowLogger wfLogger; + + public void startProcess(final WorkflowProcess process) { + log.info(process.getGraph()); + + log.info("Starting workflow: " + process); + + final LocalDateTime now = LocalDateTime.now(); + process.setStatus(WorkflowProcess.Status.EXECUTING); + process.setStartDate(now); + process.setLastActivityDate(now); + + try { + for (final GraphNode node : process.getGraph().startNodes()) { + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv()); + final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); + + token.getEnv().addAttributes(process.getEnv().getAttributes()); + process.getTokens().add(token); + + pNode.execute(token); + } + } catch (final Throwable e) { + log.error("WorkflowProcess node instantiation failed", e); + process.setStatus(WorkflowProcess.Status.FAILURE); + } + } + + public void releaseToken(final WorkflowProcess process, final GraphNode oldGraphNode, final Token oldToken) { + process.setLastActivityDate(LocalDateTime.now()); + + try { + for (final GraphNode node : process.getGraph().nextNodes(oldGraphNode, oldToken.getNextArc())) { + if (node.isJoin() || node.isSucessNode()) { + if (!process.getPausedJoinNodeTokens().containsKey(node.getName())) { + process.getPausedJoinNodeTokens().put(node.getName(), new ArrayList()); + } + + final List list = process.getPausedJoinNodeTokens().get(node.getName()); + + list.add(oldToken); + + if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) { + final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); + token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()]))); + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); + + process.getTokens().add(token); + process.setLastActivityDate(LocalDateTime.now()); + + if (node.isSucessNode()) { + markAsCompleted(process, token); + } else { + pNode.execute(token); + } + } + } else { + final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); + token.getEnv().addAttributes(oldToken.getEnv().getAttributes()); + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); + + process.getTokens().add(token); + process.setLastActivityDate(LocalDateTime.now()); + pNode.execute(token); + } + } + } catch (final Throwable e) { + log.error("WorkflowProcess node instantiation failed", e); + process.setStatus(WorkflowProcess.Status.FAILURE); + process.setError(e.getMessage()); + process.setErrorStacktrace(Throwables.getStackTraceAsString(e)); + process.setLastActivityDate(LocalDateTime.now()); + } + + } + + private NodeTokenCallback newNodeTokenCallback(final WorkflowProcess process, final GraphNode node) { + return new NodeTokenCallback() { + + @Override + public void onSuccess(final Token token) { + releaseToken(process, node, token); + } + + @Override + public void onFail(final Token token) { + completeProcess(process, token); + } + }; + } + + private Map mergeEnvParams(final Token... tokens) { + final Map map = new HashMap<>(); + Arrays.stream(tokens).forEach(t -> map.putAll(t.getEnv().getAttributes())); + return map; + } + + private void markAsCompleted(final WorkflowProcess process, final Token token) { + completeProcess(process, token); + } + + private void completeProcess(final WorkflowProcess process, final Token token) { + if (token.isActive()) { + if (StringUtils.isNotBlank(token.getError())) { + token.releaseAsFailed(token.getError()); + } else { + token.release(); + } + } + + final LocalDateTime now = token.getEndDate(); + + process.setLastActivityDate(now); + process.setEndDate(now); + process.setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS); + + if (token.isFailed()) { + process.setStatus(Status.FAILURE); + process.setError(token.getError()); + process.setErrorStacktrace(token.getErrorStackTrace()); + process.setLastActivityDate(LocalDateTime.now()); + } + + if (process.getCallback() != null) { + if (token.isFailed()) { + process.getCallback().onFail(); + } else { + process.getCallback().onSuccess(); + } + } + + final Map details = new LinkedHashMap<>(); + details.putAll(process.getOutputParams()); + details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + process.getPriority()); + details.put(WorkflowsConstants.LOG_WF_ID, process.getWfId()); + details.put(WorkflowsConstants.LOG_WF_ID, process.getWfInstanceId()); + + if (process.getError() != null) { + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, process.getError()); + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, process.getErrorStacktrace()); + } + + final WfProcessExecution pe = new WfProcessExecution(); + pe.setProcessId(process.getId()); + pe.setName(process.getName()); + pe.setFamily(process.getFamily()); + + pe.setDsId(process.getDsId()); + pe.setDsName(process.getDsName()); + pe.setDsApi(process.getDsInterface()); + + pe.setStartDate(process.getStartDate()); + pe.setEndDate(process.getEndDate()); + + pe.setStatus(process.getStatus().toString()); + + pe.setDetails(details); + + wfLogger.saveProcessExecution(pe); + + emailSender.sendMails(process); + + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java new file mode 100644 index 00000000..8cf041e3 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -0,0 +1,67 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.WorkflowDbEntry; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; +import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; + +@Component +public class ProcessFactory { + + private static final Log log = LogFactory.getLog(ProcessFactory.class); + + private String oldGeneratedId = ""; + + private final DateTimeFormatter processIdFormatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_S"); + + @Autowired + private GraphLoader graphLoader; + + public WorkflowProcess newProcess(final WorkflowDbEntry wf, + final WorkflowInstance instance, + final ProcessCallback processCallback, + final String parent) throws WorkflowManagerException { + + final Map globalParams = new HashMap<>(); + globalParams.putAll(instance.getSystemParams()); + globalParams.putAll(instance.getUserParams()); + + return new WorkflowProcess(generateProcessId(), + wf.getName(), + wf.getFamily(), + instance.getDsId(), + instance.getDsName(), + instance.getApiId(), + graphLoader.loadGraph(wf.getGraph(), globalParams), + instance.getPriority(), + wf.getId(), + instance.getId(), + globalParams, + + processCallback, parent); + + } + + private synchronized String generateProcessId() { + String id = ""; + do { + id = "wf_" + LocalDateTime.now().format(processIdFormatter); + log.info("Generated processID " + id); + } while (id.equals(oldGeneratedId)); + + oldGeneratedId = id; + + return id; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java new file mode 100644 index 00000000..45533072 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java @@ -0,0 +1,132 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.PriorityBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; + +@Service +public class ProcessRegistry { + + private static final Log log = LogFactory.getLog(ProcessRegistry.class); + private final BiMap procs = HashBiMap.create(); + private final Map> byOtherId = new HashMap<>(); + + private final PriorityBlockingQueue pendingProcs = new PriorityBlockingQueue<>(); + + @Value("${'dnet.wf.registry.size'}") + private int maxSize; + + synchronized public int countRunningWfs() { + int count = 0; + for (final Map.Entry e : this.procs.entrySet()) { + final WorkflowProcess proc = e.getValue(); + if (!proc.isTerminated()) { + count++; + } + } + return count; + } + + public WorkflowProcess findProcess(final String procId) { + return this.procs.get(procId); + } + + public Set listProcesses() { + return this.procs.values(); + } + + public Collection findProcsByOtherId(final String id) { + synchronized (this) { + final Collection res = this.byOtherId.get(id); + return res != null ? res : new ArrayList<>(); + } + } + + public String registerProcess(final WorkflowProcess process, final String... ids) throws WorkflowManagerException { + if (this.procs.containsValue(process) || this.procs.containsKey(process.getId())) { + log.error("Already registerd process: " + process); + throw new WorkflowManagerException("Already registerd process: " + process); + } + + if (this.procs.size() >= this.maxSize) { + removeOldestProcess(); + } + + this.procs.put(process.getId(), process); + for (final String id : ids) { + synchronized (this) { + if (!this.byOtherId.containsKey(id)) { + this.byOtherId.put(id, new ArrayList()); + } + this.byOtherId.get(id).add(process); + } + } + + synchronized (this.pendingProcs) { + if (this.pendingProcs.size() > WorkflowsConstants.MAX_PENDING_PROCS_SIZE) { + log.warn("Wf [" + process.getName() + "] not launched, Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE); + throw new WorkflowManagerException("Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE); + } + this.pendingProcs.put(process); + + log.info("WorkflowProcess [" + process + "] in queue, priority=" + process.getPriority()); + } + + return process.getId(); + } + + private void removeOldestProcess() { + LocalDateTime oldDate = LocalDateTime.now(); + String oldId = null; + + for (final Map.Entry e : this.procs.entrySet()) { + final WorkflowProcess proc = e.getValue(); + + if (proc.isTerminated()) { + final LocalDateTime date = proc.getLastActivityDate(); + if (date.isBefore(oldDate)) { + oldDate = date; + oldId = e.getKey(); + } + } + } + + if (oldId != null) { + unregisterProcess(oldId); + } + + } + + public void unregisterProcess(final String procId) { + synchronized (this) { + final WorkflowProcess process = this.procs.remove(procId); + if (process != null) { + for (final Collection processes : this.byOtherId.values()) { + processes.remove(process); + } + } + } + } + + public WorkflowProcess nextProcessToStart() { + synchronized (this.pendingProcs) { + return this.pendingProcs.poll(); + } + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java new file mode 100644 index 00000000..9524dfe1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -0,0 +1,147 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.UUID; + +import com.google.common.base.Throwables; + +import eu.dnetlib.manager.wf.workflows.graph.Arc; +import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback; +import eu.dnetlib.manager.wf.workflows.util.ProgressProvider; + +/** + * Created by michele on 19/11/15. + */ +public class Token { + + private final String id; + private final String nodeName; + private final Env env = new Env(); + private final LocalDateTime startDate; + private final NodeTokenCallback callback; + private boolean failed = false; + private LocalDateTime endDate = LocalDateTime.MIN; + private String nextArc = Arc.DEFAULT_ARC; + private boolean active = true; + private String error = ""; + private String errorStackTrace = ""; + private ProgressProvider progressProvider; + + public Token(final String nodeName, final NodeTokenCallback callback) { + this.id = "token-" + UUID.randomUUID(); + this.nodeName = nodeName; + this.startDate = LocalDateTime.now(); + this.callback = callback; + } + + public String getId() { + return this.id; + } + + public Env getEnv() { + return this.env; + } + + public LocalDateTime getStartDate() { + return this.startDate; + } + + public LocalDateTime getEndDate() { + return this.endDate; + } + + public void setEndDate(final LocalDateTime endDate) { + this.endDate = endDate; + } + + public String getNextArc() { + return this.nextArc; + } + + public void setNextArc(final String nextArc) { + this.nextArc = nextArc; + } + + public boolean isActive() { + return this.active; + } + + public void setActive(final boolean active) { + this.active = active; + } + + public boolean isFailed() { + return this.failed; + } + + public void setFailed(final boolean failed) { + this.failed = failed; + } + + public void release(final String arcName) { + setNextArc(arcName); + setEndDate(LocalDateTime.now()); + setActive(false); + if (this.callback != null) { + this.callback.onSuccess(this); + } + } + + public void release() { + setEndDate(LocalDateTime.now()); + setActive(false); + if (this.callback != null) { + this.callback.onSuccess(this); + } + } + + public void releaseAsFailed(final Throwable e) { + setEndDate(LocalDateTime.now()); + setActive(false); + setFailed(true); + setError(e.getMessage()); + setErrorStackTrace(Throwables.getStackTraceAsString(e)); + if (this.callback != null) { + this.callback.onFail(this); + } + } + + public void releaseAsFailed(final String error) { + setEndDate(LocalDateTime.now()); + setActive(false); + setFailed(true); + setError(error); + if (this.callback != null) { + this.callback.onFail(this); + } + } + + public String getNodeName() { + return this.nodeName; + } + + public String getError() { + return this.error; + } + + public void setError(final String error) { + this.error = error; + } + + public String getErrorStackTrace() { + return this.errorStackTrace; + } + + public void setErrorStackTrace(final String errorStackTrace) { + this.errorStackTrace = errorStackTrace; + } + + public ProgressProvider getProgressProvider() { + return this.progressProvider; + } + + public void setProgressProvider(final ProgressProvider progressProvider) { + this.progressProvider = progressProvider; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java new file mode 100644 index 00000000..525d8bb1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -0,0 +1,155 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.util.HashMap; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import eu.dnetlib.dsm.DsmService; +import eu.dnetlib.errors.DsmException; +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.WorkflowDbEntry; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.repository.WorkflowDbEntryRepository; +import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; +import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; +import eu.dnetlib.utils.Stoppable; +import eu.dnetlib.utils.StoppableDetails; + +@Service +public class WorkflowExecutor implements Stoppable { + + private static final Log log = LogFactory.getLog(WorkflowExecutor.class); + + @Autowired + private GraphLoader graphLoader; + @Autowired + private ProcessRegistry processRegistry; + @Autowired + private ProcessFactory processFactory; + @Autowired + private ProcessEngine processEngine; + @Autowired + private DsmService dsmService; + + @Autowired + private WorkflowDbEntryRepository workflowDbEntryRepository; + @Autowired + private WorkflowInstanceRepository workflowInstanceRepository; + + private boolean paused = false; + + public void init() { + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { + if (isPaused() || processRegistry.countRunningWfs() >= WorkflowsConstants.MAX_RUNNING_PROCS_SIZE) { return; } + + final WorkflowProcess process = processRegistry.nextProcessToStart(); + if (process != null) { + processEngine.startProcess(process); + } else { + log.debug("WorkflowProcess queue is empty"); + } + }, 10, 10, TimeUnit.SECONDS); + } + + public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, final ProcessCallback processCallback, final String parent) + throws WorkflowManagerException { + + if (isPaused()) { + log.warn("Wf " + wfId + " not launched, because WorkflowExecutor is preparing for shutdown"); + throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown"); + } + + try { + final String dsName = dsmService.getDs(dsId).getOfficialname(); + + final WorkflowInstance instance = new WorkflowInstance(); + instance.setId("REPO_HI_" + UUID.randomUUID()); + instance.setDetails(new HashMap<>()); + instance.setPriority(100); + instance.setDsId(dsId); + instance.setDsName(dsName); + instance.setApiId(apiId); + instance.setEnabled(true); + instance.setConfigured(true); + instance.setSchedulingEnabled(false); + instance.setCronExpression(""); + instance.setCronMinInterval(0); + instance.setWorkflow(wfId); + instance.setDestroyWf(null); + instance.setSystemParams(new HashMap<>()); + instance.setUserParams(new HashMap<>()); + + return startWorkflowInstance(instance, processCallback, parent); + } catch (final DsmException e) { + throw new WorkflowManagerException("Invalid datasource: " + dsId, e); + } + } + + public String startWorkflowInstance(final String wfInstanceId, final ProcessCallback processCallback, final String parent) throws Exception { + + if (isPaused()) { + log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown"); + throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown"); + } + + final WorkflowInstance instance = + workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId)); + return startWorkflowInstance(instance, processCallback, parent); + } + + public String startWorkflowInstance(final WorkflowInstance instance, final ProcessCallback processCallback, final String parent) + throws WorkflowManagerException { + final WorkflowDbEntry wf = workflowDbEntryRepository.findById(instance.getWorkflow()) + .orElseThrow(() -> new WorkflowManagerException("WF not found: " + instance.getWorkflow())); + + if (!instance.isEnabled() || !instance.isConfigured()) { + log.warn("Wf instance " + instance.getId() + " not launched, because it is not ready to start or it is disabled"); + throw new WorkflowManagerException("Workflow " + instance.getId() + " is not ready to start"); + } + + final WorkflowProcess process = + processFactory.newProcess(wf, instance, processCallback, parent); + + return processRegistry.registerProcess(process, instance.getId()); + } + + @Override + public void stop() { + paused = true; + } + + @Override + public void resume() { + paused = false; + } + + @Override + public StoppableDetails getStopDetails() { + final int count = processRegistry.countRunningWfs(); + + final StoppableDetails.StopStatus status = + isPaused() ? count == 0 ? StoppableDetails.StopStatus.STOPPED : StoppableDetails.StopStatus.STOPPING : StoppableDetails.StopStatus.RUNNING; + + return new StoppableDetails("D-NET workflow manager", "Running workflows: " + count, status); + } + + public ProcessRegistry getProcessRegistry() { + return processRegistry; + } + + public boolean isPaused() { + return paused; + } + + public void setPaused(final boolean paused) { + this.paused = paused; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java new file mode 100644 index 00000000..9a479806 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -0,0 +1,258 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.lang3.math.NumberUtils; + +import eu.dnetlib.manager.wf.workflows.graph.Graph; +import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; + +/** + * Created by michele on 19/11/15. + */ +public class WorkflowProcess implements Comparable { + + private final String id; + private final String name; + private final String family; + private String dsId; + private String dsName; + private String dsInterface; + private final Graph graph; + private final ProcessCallback callback; + private final int priority; + private final Env env; + private final List tokens = new CopyOnWriteArrayList<>(); + private LocalDateTime lastActivityDate; + private Status status; + private LocalDateTime startDate = LocalDateTime.MIN; + private LocalDateTime endDate = LocalDateTime.MIN; + private final String wfId; + private final String wfInstanceId; + private Map> pausedJoinNodeTokens = new HashMap<>(); + private Map globalParams; + private String error; + private String errorStacktrace; + private Map outputParams = new HashMap<>(); + private String parentProfileId; + + public WorkflowProcess( + final String id, + final String name, + final String family, + final String dsId, + final String dsName, + final String dsInterface, + final Graph graph, + final int priority, + final String wfId, + final String wfInstanceId, + final Map globalParams, + final ProcessCallback callback, + final String parentProfileId) { + this.id = id; + this.name = name; + this.family = family; + this.dsId = dsId; + this.dsName = dsName; + this.dsInterface = dsInterface; + this.graph = graph; + this.priority = priority; + this.callback = callback; + this.status = Status.CREATED; + this.env = new Env(); + this.wfId = wfId; + this.wfInstanceId = wfInstanceId; + this.globalParams = globalParams; + this.lastActivityDate = LocalDateTime.now(); + this.parentProfileId = parentProfileId; + } + + public void setParentProfileId(final String parentProfileId) { + this.parentProfileId = parentProfileId; + } + + public String getDsId() { + return dsId; + } + + public void setDsId(final String dsId) { + this.dsId = dsId; + } + + public String getDsName() { + return dsName; + } + + public void setDsName(final String dsName) { + this.dsName = dsName; + } + + public String getDsInterface() { + return dsInterface; + } + + public void setDsInterface(final String dsInterface) { + this.dsInterface = dsInterface; + } + + public Map> getPausedJoinNodeTokens() { + return pausedJoinNodeTokens; + } + + public void setPausedJoinNodeTokens(final Map> pausedJoinNodeTokens) { + this.pausedJoinNodeTokens = pausedJoinNodeTokens; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public Env getEnv() { + return env; + } + + public Status getStatus() { + return status; + } + + public void setStatus(final Status status) { + this.status = status; + } + + public Graph getGraph() { + return graph; + } + + public List getTokens() { + return tokens; + } + + public void kill() { + setStatus(Status.KILLED); + } + + public ProcessCallback getCallback() { + return callback; + } + + public int getPriority() { + return priority; + } + + public boolean isTerminated() { + switch (status) { + case SUCCESS: + case FAILURE: + case KILLED: + return true; + default: + return false; + } + } + + public LocalDateTime getLastActivityDate() { + return lastActivityDate; + } + + public void setLastActivityDate(final LocalDateTime lastActivityDate) { + this.lastActivityDate = lastActivityDate; + } + + @Override + public String toString() { + return String.format("[process id='%s' name='%s']", id, name); + } + + @Override + public int compareTo(final WorkflowProcess wp) { + return NumberUtils.compare(getPriority(), wp.getPriority()); + } + + public Map getGlobalParams() { + return globalParams; + } + + public void setGlobalParams(final Map globalParams) { + this.globalParams = globalParams; + } + + public String getFamily() { + return family; + } + + public String getWfId() { + return wfId; + } + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setStartDate(final LocalDateTime startDate) { + this.startDate = startDate; + } + + public void setEndDate(final LocalDateTime endDate) { + this.endDate = endDate; + } + + public LocalDateTime getStartDate() { + return startDate; + } + + public LocalDateTime getEndDate() { + return endDate; + } + + public enum Status { + CREATED, + EXECUTING, + SUCCESS, + FAILURE, + KILLED; + } + + public enum StartMode { + AUTO, + MANUAL, + DISABLED + } + + public String getError() { + return error; + } + + public void setError(final String error) { + this.error = error; + } + + public String getErrorStacktrace() { + return errorStacktrace; + } + + public void setErrorStacktrace(final String errorStacktrace) { + this.errorStacktrace = errorStacktrace; + } + + public void setOutputParams(final Map outputParams) { + this.outputParams = outputParams; + } + + public Map getOutputParams() { + return outputParams; + } + + public String getParentProfileId() { + return parentProfileId; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java new file mode 100644 index 00000000..35cf72ce --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java @@ -0,0 +1,66 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.PropertyAccessorFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.nodes.DefaultJobNode; +import eu.dnetlib.manager.wf.nodes.ProcessNode; +import eu.dnetlib.manager.wf.nodes.SuccessNode; +import eu.dnetlib.manager.wf.workflows.graph.GraphNode; +import eu.dnetlib.manager.wf.workflows.procs.Env; +import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; + +/** + * Created by michele on 19/11/15. + */ +public class NodeHelper implements ApplicationContextAware { + + public static final String beanNamePrefix = "wfNode"; + private static final Log log = LogFactory.getLog(NodeHelper.class); + + private ApplicationContext applicationContext; + + public ProcessNode newProcessNode(final GraphNode node, final WorkflowProcess process, final Env env) throws WorkflowManagerException { + if (node.isSucessNode()) { + return new SuccessNode(); + } else if (StringUtils.isBlank(node.getType())) { + return new DefaultJobNode(node.getName()); + } else { + final ProcessNode pnode = this.applicationContext.getBean(beanNamePrefix + node.getType(), ProcessNode.class); + if (pnode != null) { + pnode.setNodeName(node.getName()); + // I invoke the setter methods using the static params of the graph node + try { + PropertyAccessorFactory.forBeanPropertyAccess(pnode).setPropertyValues(node.resolveParams(env)); + } catch (final Throwable e) { + throw new WorkflowManagerException(String.format("error setting parameters in wfNode %s", node.getName()), e); + } + if (pnode instanceof ProcessAware) { + ((ProcessAware) pnode).setProcess(process); + } + return pnode; + } else { + log.error("cannot find bean " + beanNamePrefix + node.getType()); + throw new WorkflowManagerException("cannot find bean " + beanNamePrefix + node.getType()); + } + } + } + + public boolean isValidType(final String type) { + return StringUtils.isBlank(type) || this.applicationContext.isPrototype(beanNamePrefix + type) && this.applicationContext + .isTypeMatch(beanNamePrefix + type, ProcessNode.class); + } + + @Override + public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java new file mode 100644 index 00000000..7496186b --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java @@ -0,0 +1,13 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 26/11/15. + */ +public interface NodeTokenCallback { + + void onSuccess(Token token); + + void onFail(Token token); +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java new file mode 100644 index 00000000..4f70b5da --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java @@ -0,0 +1,12 @@ +package eu.dnetlib.manager.wf.workflows.util; + +/** + * Created by michele on 18/11/15. + */ +public interface ProcessCallback { + + void onSuccess(); + + void onFail(); + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java new file mode 100644 index 00000000..d9f84f45 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java @@ -0,0 +1,6 @@ +package eu.dnetlib.manager.wf.workflows.util; + +public interface ProgressProvider { + + String getProgressDescription(); +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java new file mode 100644 index 00000000..499349a3 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java @@ -0,0 +1,37 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import java.util.List; +import java.util.stream.Collectors; + +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; + +public class SubWorkflowProgressProvider implements ProgressProvider { + + private final String procId; + private final ProcessRegistry processRegistry; + + public SubWorkflowProgressProvider(final String procId, final ProcessRegistry processRegistry) { + super(); + this.procId = procId; + this.processRegistry = processRegistry; + } + + @Override + public String getProgressDescription() { + final WorkflowProcess proc = this.processRegistry.findProcess(this.procId); + + if (proc == null) { return "-"; } + + final List list = proc.getTokens() + .stream() + .filter(t -> t.isActive()) + .map(t -> t.getProgressProvider() != null ? String.format("%s (%s)", t.getNodeName(), t.getProgressProvider().getProgressDescription()) + : t.getNodeName()) + .collect(Collectors.toList()); + if (!list.isEmpty()) { return list.stream().collect(Collectors.joining(", ")); } + + return "-"; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java new file mode 100644 index 00000000..956abac4 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java @@ -0,0 +1,71 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Required; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.workflows.util.ValidNodeValuesFetcher.DnetParamValue; + +public abstract class ValidNodeValuesFetcher implements Function, List> { + + private String name; + + private static final Log log = LogFactory.getLog(ValidNodeValuesFetcher.class); + + public class DnetParamValue implements Comparable { + + private final String id; + private final String name; + + public DnetParamValue(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return this.id; + } + + public String getName() { + return this.name; + } + + @Override + public int compareTo(final DnetParamValue o) { + return getName().compareTo(o.getName()); + } + } + + @Override + final public List apply(final Map params) { + try { + return obtainValues(params); + } catch (final Throwable e) { + log.error("Error obtaing values", e); + return new ArrayList<>(); + } + } + + abstract protected List obtainValues(Map params) throws Exception; + + public String getName() { + return this.name; + } + + @Required + public void setName(final String name) { + this.name = name; + } + + protected void verifyParams(final Map params, final String... pnames) throws WorkflowManagerException { + for (final String s : pnames) { + if (!params.containsKey(s)) { throw new WorkflowManagerException("Parameter not found: " + s); } + } + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java new file mode 100644 index 00000000..3be95afc --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java @@ -0,0 +1,119 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Required; + +import com.google.gson.Gson; + +public class WorkflowsConstants { + + public static final String DATASOURCE_PREFIX = "datasource:"; + + public static final String LOG_WF_NAME = "system:wfName"; + public static final String LOG_WF_ID = "system:wfId"; + public static final String LOG_WF_INSTANCE_ID = "system:wfInstanceId"; + + public static final String LOG_WF_FAMILY = "system:family"; + public static final String LOG_WF_PRIORITY = "system:priority"; + public static final String LOG_WF_PROCESS_ID = "system:processId"; + public static final String LOG_WF_PROCESS_STATUS = "system:processStatus"; + public static final String LOG_WF_PROCESS_START_DATE = "system:startDate"; + public static final String LOG_WF_PROCESS_END_DATE = "system:endDate"; + public static final String LOG_WF_PARENT = "system:parentProfileId"; + + public static final String LOG_SYSTEM_ERROR = "system:error"; + public static final String LOG_SYSTEM_ERROR_STACKTRACE = "system:error:stacktrace"; + + public static final String LOG_DATASOURCE_NAME = WorkflowsConstants.DATASOURCE_PREFIX + "name"; + public static final String LOG_DATASOURCE_ID = WorkflowsConstants.DATASOURCE_PREFIX + "id"; + public static final String LOG_DATASOURCE_INTERFACE = WorkflowsConstants.DATASOURCE_PREFIX + "interface"; + + public static final String BLACKBOARD_IS_BLACKBOARD = "blackboard:isBlackboard"; + public static final String BLACKBOARD_JOB = "blackboard:job"; + public static final String BLACKBOARD_SERVICE_ID = "blackboard:serviceId"; + public static final String BLACKBOARD_IS_GOING = "blackboard:isOngoing"; + public static final String BLACKBOARD_PARAM_PREFIX = "blackboard:param:"; + + // public static final String DATASOURCE_ACRONYM = WorkflowsConstants.DATASOURCE_PREFIX + "acronym"; + // public static final String DATASOURCE_URL = WorkflowsConstants.DATASOURCE_PREFIX + "url"; + + public static final int MIN_WF_PRIORITY = 0; + public static final int MAX_WF_PRIORITY = 100; + public static final int DEFAULT_WF_PRIORITY = 50; + public static final int MAX_PENDING_PROCS_SIZE = 100; + public static final int MAX_RUNNING_PROCS_SIZE = 100; + + public static final String MAIN_LOG_PREFIX = "mainlog:"; + + private String datasourceProtocolsJson; + private String datasourceTypologiesJson; + private List> datasourceProtocols; + private List> datasourceTypologies; + private List> datasourceWorkflowStatuses; + + @SuppressWarnings("unchecked") + public void init() { + final Gson gson = new Gson(); + datasourceProtocols = gson.fromJson(datasourceProtocolsJson, List.class); + datasourceTypologies = gson.fromJson(datasourceTypologiesJson, List.class); + datasourceWorkflowStatuses = new ArrayList<>(); + for (final WorkflowStatus s : WorkflowStatus.values()) { + final Map map = new HashMap<>(); + map.put("name", s.displayName); + map.put("icon", s.icon); + map.put("value", s.toString()); + datasourceWorkflowStatuses.add(map); + } + } + + public String getDatasourceProtocolsJson() { + return datasourceProtocolsJson; + } + + @Required + public void setDatasourceProtocolsJson(final String datasourceProtocolsJson) { + this.datasourceProtocolsJson = datasourceProtocolsJson; + } + + public String getDatasourceTypologiesJson() { + return datasourceTypologiesJson; + } + + @Required + public void setDatasourceTypologiesJson(final String datasourceTypologiesJson) { + this.datasourceTypologiesJson = datasourceTypologiesJson; + } + + public List> getDatasourceProtocols() { + return datasourceProtocols; + } + + public List> getDatasourceTypologies() { + return datasourceTypologies; + } + + public List> getDatasourceWorkflowStatuses() { + return datasourceWorkflowStatuses; + } + + public enum WorkflowStatus { + + EXECUTABLE("Executable", "icon-ok"), + WAIT_USER_SETTINGS("Waiting user settings", "icon-edit"), + WAIT_SYS_SETTINGS("Waiting system settings", + "icon-refresh"); + + public String displayName; + public String icon; + + WorkflowStatus(final String displayName, final String icon) { + this.displayName = displayName; + this.icon = icon; + } + } + +} diff --git a/libs/pom.xml b/libs/pom.xml index e7256d31..20ba7d51 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -19,6 +19,7 @@ dnet-is-common dnet-is-services dnet-data-services + dnet-wf-service diff --git a/pom.xml b/pom.xml index b7839b62..2de8e485 100644 --- a/pom.xml +++ b/pom.xml @@ -347,8 +347,8 @@ maven-compiler-plugin ${maven.compiler.plugin.version} - 1.8 - 1.8 + 17 + 17 ${project.build.sourceEncoding} @@ -451,7 +451,7 @@ UTF-8 UTF-8 - 3.6.0 + 3.9.0 1.8 2.14.0 7.1.0 From 01b046cfb42f761effe8def4f1caaaafb9104ca7 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 27 Feb 2023 13:09:58 +0100 Subject: [PATCH 02/19] some fix --- .../src/main/resources/application.properties | 11 ++++++++++- .../eu/dnetlib/data/mdstore/model/MDStore.java | 3 --- .../data/mdstore/model/MDStoreVersion.java | 3 --- .../data/mdstore/model/MDStoreWithInfo.java | 4 ---- .../is/model/resource/SimpleResource.java | 4 ---- .../manager/wf/model/WfProcessExecution.java | 4 ---- .../manager/wf/model/WorkflowInstance.java | 6 ++++-- .../notifications/mail/EmailDispatcher.java | 18 +++++++++--------- 8 files changed, 23 insertions(+), 30 deletions(-) diff --git a/apps/dnet-is-application/src/main/resources/application.properties b/apps/dnet-is-application/src/main/resources/application.properties index f56fd65d..a2a29b36 100644 --- a/apps/dnet-is-application/src/main/resources/application.properties +++ b/apps/dnet-is-application/src/main/resources/application.properties @@ -3,6 +3,8 @@ server.port=8280 server.public_url = server.public_desc = API Base URL +dnet.configuration.infrastructure = LOCAL DEV + spring.profiles.active=dev maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-pom.xml @@ -57,5 +59,12 @@ dhp.mdstore-manager.hadoop.zeppelin.login = dhp.mdstore-manager.hadoop.zeppelin.password = dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager - +# Email Configuration +dnet.configuration.mail.sender.email = +dnet.configuration.mail.sender.name = +dnet.configuration.mail.cc = +dnet.configuration.mail.smtp.host = +dnet.configuration.mail.smtp.port = 25 +dnet.configuration.mail.smtp.user = +dnet.configuration.mail.smtp.password = diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java index aaf488da..ff71cda7 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java @@ -12,8 +12,6 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -55,7 +53,6 @@ public class MDStore implements Serializable { private Map params = new LinkedHashMap<>(); @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) private LocalDateTime creationDate; @Override diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java index 4ba17ffa..29e958d5 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java @@ -10,8 +10,6 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -36,7 +34,6 @@ public class MDStoreVersion implements Serializable { private int readCount = 0; @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) private LocalDateTime lastUpdate; @Column(name = "size") diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java index b95133f1..ff570971 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java @@ -12,8 +12,6 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -54,11 +52,9 @@ public class MDStoreWithInfo implements Serializable { private String currentVersion; @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) private LocalDateTime creationDate; @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) private LocalDateTime lastUpdate; @Column(name = "size") diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java index a2ec6c79..b585512f 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java @@ -7,8 +7,6 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; @Entity @Table(name = "resources") @@ -29,11 +27,9 @@ public class SimpleResource implements Serializable { @Column(name = "description") private String description; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "creation_date") private LocalDateTime creationDate; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "modification_date") private LocalDateTime modificationDate; diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java index 4863e27d..4bafb574 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java @@ -8,8 +8,6 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; @@ -44,11 +42,9 @@ public class WfProcessExecution implements Serializable { @Column(name = "status") private String status; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "start_date") private LocalDateTime startDate; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "end_date") private LocalDateTime endDate; diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java index a0ddca0a..53d5afd1 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -67,10 +67,12 @@ public class WorkflowInstance implements Serializable { @Column(name = "destroy_wf") private String destroyWf; - @Column(name = "system_params") + @Type(type = "jsonb") + @Column(name = "system_params", columnDefinition = "jsonb") private Map systemParams = new LinkedHashMap<>(); - @Column(name = "user_params") + @Type(type = "jsonb") + @Column(name = "user_params", columnDefinition = "jsonb") private Map userParams = new LinkedHashMap<>(); public String getId() { diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java index c5396b69..4a0afec7 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java @@ -33,23 +33,23 @@ public class EmailDispatcher { private static final Log log = LogFactory.getLog(EmailDispatcher.class); private final BlockingQueue queue = new LinkedBlockingQueue<>(); - @Value("${'dnet.configuration.mail.sender.email'}") + @Value("${dnet.configuration.mail.sender.email}") private String from; - @Value("${'dnet.configuration.mail.sender.name'}") + @Value("${dnet.configuration.mail.sender.name}") private String fromName; - @Value("${'dnet.configuration.mail.cc'}") + @Value("${dnet.configuration.mail.cc}") private String cc; - @Value("${'dnet.configuration.mail.smtp.host'}") + @Value("${dnet.configuration.mail.smtp.host}") private String smtpHost; - @Value("${'dnet.configuration.mail.smtp.port'}") + @Value("${dnet.configuration.mail.smtp.port}") private final int smtpPort = 587; - @Value("${'dnet.configuration.mail.smtp.user'}") + @Value("${dnet.configuration.mail.smtp.user}") private String smtpUser; - @Value("${'dnet.configuration.mail.smtp.password'}") + @Value("${dnet.configuration.mail.smtp.password}") private String smtpPassword; - @Value("${'dnet.configuration.baseUrl'}") + @Value("${server.public_url}") private String baseUrl; - @Value("${'dnet.configuration.infrastructure'}") + @Value("${dnet.configuration.infrastructure}") private String infrastructure; @Autowired From a3d8b0de4d90711a8cf50ff6b6aa49ef63fe19eb Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 1 Mar 2023 13:01:24 +0100 Subject: [PATCH 03/19] some fixes --- apps/dnet-is-application/pom.xml | 2 +- .../src/main/java/eu/dnetlib/MainDBConfig.java | 10 ++++++++-- .../dnetlib/data/mdstore/ZeppelinAjaxController.java | 1 - .../java/eu/dnetlib/is/importer/WfHistoryImporter.java | 4 ++-- .../eu/dnetlib/manager/wf/WfHistoryAjaxController.java | 3 ++- .../src/main/resources/application.properties | 2 +- .../{manager/wf => common}/model/EmailTemplate.java | 2 +- .../{wf => history}/model/WfProcessExecution.java | 2 +- .../repository/EmailTemplateRepository.java | 4 ++-- .../manager/{wf => history}/WorkflowLogger.java | 6 +++--- .../repository/WfProcessExecutionRepository.java | 4 ++-- .../eu/dnetlib/notifications/mail/EmailDispatcher.java | 2 +- .../manager/wf/cron/ScheduledWorkflowLauncher.java | 2 +- .../manager/wf/workflows/procs/ProcessEngine.java | 4 ++-- .../manager/wf/workflows/procs/WorkflowExecutor.java | 6 +++--- pom.xml | 2 +- 16 files changed, 31 insertions(+), 25 deletions(-) rename libs/dnet-is-common/src/main/java/eu/dnetlib/{manager/wf => common}/model/EmailTemplate.java (95%) rename libs/dnet-is-common/src/main/java/eu/dnetlib/manager/{wf => history}/model/WfProcessExecution.java (98%) rename libs/dnet-is-services/src/main/java/eu/dnetlib/{manager/wf => common}/repository/EmailTemplateRepository.java (62%) rename libs/dnet-is-services/src/main/java/eu/dnetlib/manager/{wf => history}/WorkflowLogger.java (90%) rename libs/dnet-is-services/src/main/java/eu/dnetlib/manager/{wf => history}/repository/WfProcessExecutionRepository.java (80%) diff --git a/apps/dnet-is-application/pom.xml b/apps/dnet-is-application/pom.xml index f563dabe..27cb2137 100644 --- a/apps/dnet-is-application/pom.xml +++ b/apps/dnet-is-application/pom.xml @@ -25,7 +25,7 @@ dnet-data-services ${project.version} - + org.springframework.boot diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java index 5978f4ac..882363e9 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java @@ -20,7 +20,11 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration @EnableTransactionManagement @EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = { - "eu.dnetlib.is", "eu.dnetlib.data.mdstore", "eu.dnetlib.manager" + "eu.dnetlib.is", + "eu.dnetlib.common", + "eu.dnetlib.manager.history", + // "eu.dnetlib.manager.wf", + "eu.dnetlib.data.mdstore" }) public class MainDBConfig { @@ -38,7 +42,9 @@ public class MainDBConfig { @Qualifier("mainDataSource") final DataSource ds) { return builder .dataSource(ds) - .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model") + .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", + // "eu.dnetlib.manager.wf.model", + "eu.dnetlib.data.mdstore.model") .persistenceUnit("is") .build(); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java index 55a4590e..d5ff171b 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java @@ -21,7 +21,6 @@ public class ZeppelinAjaxController extends AbstractDnetController { @GetMapping("/templates") public List getTemplates() throws MDStoreManagerException { try { - // if (zeppelinClient.get) return zeppelinClient.listTemplates(); } catch (final Throwable e) { throw new MDStoreManagerException("Zeppelin is unreachable", e); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java index a1eaa4b0..1a6c6b27 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java @@ -19,8 +19,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.manager.wf.model.WfProcessExecution; -import eu.dnetlib.manager.wf.repository.WfProcessExecutionRepository; +import eu.dnetlib.manager.history.model.WfProcessExecution; +import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository; @Service public class WfHistoryImporter { diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java index cccf4db3..dc4b18f5 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java @@ -9,7 +9,8 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.history.model.WfProcessExecution; @RestController @RequestMapping("/ajax/wfs") diff --git a/apps/dnet-is-application/src/main/resources/application.properties b/apps/dnet-is-application/src/main/resources/application.properties index a2a29b36..1c0c6fc5 100644 --- a/apps/dnet-is-application/src/main/resources/application.properties +++ b/apps/dnet-is-application/src/main/resources/application.properties @@ -11,7 +11,7 @@ maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-po spring.main.banner-mode = off -logging.level.root = INFO +logging.level.root = WARN #logging.level.org.springframework = DEBUG management.endpoints.web.exposure.include = prometheus,health diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java similarity index 95% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java index 0ba91984..bd1a2849 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/EmailTemplate.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java @@ -1,4 +1,4 @@ -package eu.dnetlib.manager.wf.model; +package eu.dnetlib.common.model; import java.io.Serializable; diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java similarity index 98% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java index 4bafb574..cbd78045 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WfProcessExecution.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java @@ -1,4 +1,4 @@ -package eu.dnetlib.manager.wf.model; +package eu.dnetlib.manager.history.model; import java.io.Serializable; import java.time.LocalDateTime; diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java similarity index 62% rename from libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java rename to libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java index 5e98cf23..f1ce156b 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/EmailTemplateRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java @@ -1,8 +1,8 @@ -package eu.dnetlib.manager.wf.repository; +package eu.dnetlib.common.repository; import org.springframework.data.jpa.repository.JpaRepository; -import eu.dnetlib.manager.wf.model.EmailTemplate; +import eu.dnetlib.common.model.EmailTemplate; public interface EmailTemplateRepository extends JpaRepository { diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java similarity index 90% rename from libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java rename to libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java index 31ae50cd..c1dd8fd8 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/WorkflowLogger.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java @@ -1,4 +1,4 @@ -package eu.dnetlib.manager.wf; +package eu.dnetlib.manager.history; import java.time.Instant; import java.time.LocalDateTime; @@ -11,8 +11,8 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; -import eu.dnetlib.manager.wf.model.WfProcessExecution; -import eu.dnetlib.manager.wf.repository.WfProcessExecutionRepository; +import eu.dnetlib.manager.history.model.WfProcessExecution; +import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository; @Service public class WorkflowLogger { diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java similarity index 80% rename from libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java rename to libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java index cf58142d..0ea555d0 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WfProcessExecutionRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java @@ -1,4 +1,4 @@ -package eu.dnetlib.manager.wf.repository; +package eu.dnetlib.manager.history.repository; import java.time.LocalDateTime; import java.util.List; @@ -6,7 +6,7 @@ import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; -import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.history.model.WfProcessExecution; public interface WfProcessExecutionRepository extends JpaRepository { diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java index 4a0afec7..e3a1e90b 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java @@ -25,7 +25,7 @@ import org.springframework.stereotype.Service; import com.google.common.base.Splitter; -import eu.dnetlib.manager.wf.repository.EmailTemplateRepository; +import eu.dnetlib.common.repository.EmailTemplateRepository; @Service public class EmailDispatcher { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 4f07469c..3711a3e1 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -11,7 +11,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.support.CronExpression; import org.springframework.stereotype.Service; -import eu.dnetlib.manager.wf.WorkflowLogger; +import eu.dnetlib.manager.history.WorkflowLogger; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java index 64beaccd..16c8bc50 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -16,8 +16,8 @@ import org.springframework.stereotype.Service; import com.google.common.base.Throwables; -import eu.dnetlib.manager.wf.WorkflowLogger; -import eu.dnetlib.manager.wf.model.WfProcessExecution; +import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.history.model.WfProcessExecution; import eu.dnetlib.manager.wf.nodes.ProcessNode; import eu.dnetlib.manager.wf.notification.EmailSender; import eu.dnetlib.manager.wf.workflows.graph.GraphNode; diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java index 525d8bb1..44ea7223 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -5,6 +5,8 @@ import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -17,7 +19,6 @@ import eu.dnetlib.manager.wf.model.WorkflowDbEntry; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowDbEntryRepository; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; -import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; import eu.dnetlib.utils.Stoppable; @@ -28,8 +29,6 @@ public class WorkflowExecutor implements Stoppable { private static final Log log = LogFactory.getLog(WorkflowExecutor.class); - @Autowired - private GraphLoader graphLoader; @Autowired private ProcessRegistry processRegistry; @Autowired @@ -46,6 +45,7 @@ public class WorkflowExecutor implements Stoppable { private boolean paused = false; + @PostConstruct public void init() { Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { if (isPaused() || processRegistry.countRunningWfs() >= WorkflowsConstants.MAX_RUNNING_PROCS_SIZE) { return; } diff --git a/pom.xml b/pom.xml index 2de8e485..2047af42 100644 --- a/pom.xml +++ b/pom.xml @@ -451,7 +451,7 @@ UTF-8 UTF-8 - 3.9.0 + 3.11.0 1.8 2.14.0 7.1.0 From f0e780eabffc64141b35cf6023e2698bf2de9f9e Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 1 Mar 2023 13:53:46 +0100 Subject: [PATCH 04/19] fix a bug --- .../src/main/java/eu/dnetlib/MainDBConfig.java | 6 ++---- .../src/main/resources/application.properties | 2 +- .../java/eu/dnetlib/notifications/mail/EmailDispatcher.java | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java index 882363e9..d3b01f91 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java @@ -23,7 +23,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; "eu.dnetlib.is", "eu.dnetlib.common", "eu.dnetlib.manager.history", - // "eu.dnetlib.manager.wf", + "eu.dnetlib.manager.wf", "eu.dnetlib.data.mdstore" }) public class MainDBConfig { @@ -42,9 +42,7 @@ public class MainDBConfig { @Qualifier("mainDataSource") final DataSource ds) { return builder .dataSource(ds) - .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", - // "eu.dnetlib.manager.wf.model", - "eu.dnetlib.data.mdstore.model") + .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model") .persistenceUnit("is") .build(); } diff --git a/apps/dnet-is-application/src/main/resources/application.properties b/apps/dnet-is-application/src/main/resources/application.properties index 1c0c6fc5..a2a29b36 100644 --- a/apps/dnet-is-application/src/main/resources/application.properties +++ b/apps/dnet-is-application/src/main/resources/application.properties @@ -11,7 +11,7 @@ maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-po spring.main.banner-mode = off -logging.level.root = WARN +logging.level.root = INFO #logging.level.org.springframework = DEBUG management.endpoints.web.exposure.include = prometheus,health diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java index e3a1e90b..0c6a2756 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java @@ -75,7 +75,7 @@ public class EmailDispatcher { throw new RuntimeException(e1); } } - }).run(); + }).start(); } public void sendMail(final String to, final String subject, final String message) { From 39f8b209551b13144c53b3cd07be1b43703a3cc7 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 1 Mar 2023 14:35:54 +0100 Subject: [PATCH 05/19] updated the version of some maven plugins --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 2047af42..f66fa57e 100644 --- a/pom.xml +++ b/pom.xml @@ -362,7 +362,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.2.1 attach-sources @@ -377,7 +377,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M4 + 3.0.0-M9 true @@ -385,7 +385,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.5.0 true none @@ -394,7 +394,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.0 + 3.5.0 From 0a70952cda1203a67b1b76bad70b6ca2e000b514 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 2 Mar 2023 12:24:38 +0100 Subject: [PATCH 06/19] new angular components --- .../dnet-is-application/src/app/app-routing.module.ts | 4 ++++ frontends/dnet-is-application/src/app/app.module.ts | 6 +++++- .../src/app/emails/emails.component.css | 0 .../src/app/emails/emails.component.html | 1 + .../src/app/emails/emails.component.ts | 10 ++++++++++ .../main-menu-panels/main-menu-panels.component.html | 9 +++++++++ .../dnet-is-application/src/app/wfs/wfs.component.css | 0 .../dnet-is-application/src/app/wfs/wfs.component.html | 1 + .../dnet-is-application/src/app/wfs/wfs.component.ts | 10 ++++++++++ libs/dnet-is-common/src/main/resources/sql/schema.sql | 8 ++++++++ 10 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 frontends/dnet-is-application/src/app/emails/emails.component.css create mode 100644 frontends/dnet-is-application/src/app/emails/emails.component.html create mode 100644 frontends/dnet-is-application/src/app/emails/emails.component.ts create mode 100644 frontends/dnet-is-application/src/app/wfs/wfs.component.css create mode 100644 frontends/dnet-is-application/src/app/wfs/wfs.component.html create mode 100644 frontends/dnet-is-application/src/app/wfs/wfs.component.ts diff --git a/frontends/dnet-is-application/src/app/app-routing.module.ts b/frontends/dnet-is-application/src/app/app-routing.module.ts index ae8898c5..3401870c 100644 --- a/frontends/dnet-is-application/src/app/app-routing.module.ts +++ b/frontends/dnet-is-application/src/app/app-routing.module.ts @@ -9,6 +9,8 @@ import { ContextViewerComponent, ContextsComponent } from './contexts/contexts.c import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/dsm.component'; import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; +import { EmailsComponent } from './emails/emails.component'; +import { WfsComponent } from './wfs/wfs.component'; const routes: Routes = [ { path: "", redirectTo: 'info', pathMatch: 'full' }, @@ -17,6 +19,8 @@ const routes: Routes = [ { path: "adv_resources/context", component: ContextsComponent }, { path: "adv_resources/vocabulary", component: VocabulariesComponent }, { path: "adv_resources/protocol", component: ProtocolsComponent }, + { path: "adv_resources/email", component: EmailsComponent }, + { path: "wfs", component: WfsComponent }, { path: "wf_history", component: WfHistoryComponent }, { path: "ctx_viewer", component: ContextViewerComponent }, { path: "voc_editor", component: VocabularyEditorComponent }, diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 5ae2c16d..22bea7a2 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -36,6 +36,8 @@ import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { SpinnerHttpInterceptor } from './common/spinner.service'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; +import { EmailsComponent } from './emails/emails.component'; +import { WfsComponent } from './wfs/wfs.component'; @NgModule({ declarations: [ @@ -66,7 +68,9 @@ import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.componen MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog, - CleanerTesterComponent + CleanerTesterComponent, + EmailsComponent, + WfsComponent ], imports: [ BrowserModule, diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.css b/frontends/dnet-is-application/src/app/emails/emails.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.html b/frontends/dnet-is-application/src/app/emails/emails.component.html new file mode 100644 index 00000000..57e79d74 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/emails.component.html @@ -0,0 +1 @@ +

emails works!

diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.ts b/frontends/dnet-is-application/src/app/emails/emails.component.ts new file mode 100644 index 00000000..166974f6 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/emails.component.ts @@ -0,0 +1,10 @@ +import { Component } from '@angular/core'; + +@Component({ + selector: 'app-emails', + templateUrl: './emails.component.html', + styleUrls: ['./emails.component.css'] +}) +export class EmailsComponent { + +} diff --git a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html index 416d80dd..d5d864d7 100644 --- a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html +++ b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html @@ -51,6 +51,15 @@ + + + Workflows + +
+ Workflows +
+ + Tools diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.css b/frontends/dnet-is-application/src/app/wfs/wfs.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.html b/frontends/dnet-is-application/src/app/wfs/wfs.component.html new file mode 100644 index 00000000..87d7cd4b --- /dev/null +++ b/frontends/dnet-is-application/src/app/wfs/wfs.component.html @@ -0,0 +1 @@ +

wfs works!

diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.ts b/frontends/dnet-is-application/src/app/wfs/wfs.component.ts new file mode 100644 index 00000000..7b12ca91 --- /dev/null +++ b/frontends/dnet-is-application/src/app/wfs/wfs.component.ts @@ -0,0 +1,10 @@ +import { Component } from '@angular/core'; + +@Component({ + selector: 'app-wfs', + templateUrl: './wfs.component.html', + styleUrls: ['./wfs.component.css'] +}) +export class WfsComponent { + +} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index c734c2e3..18a60177 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -178,6 +178,14 @@ CREATE VIEW resource_types_view AS ( count(*) AS count, false AS simple FROM protocols +) UNION ALL ( + SELECT + 'email' AS id, + 'Email templates' AS name, + 'text/plain' AS content_type, + count(*) AS count, + false AS simple + FROM emails ); CREATE TABLE mdstores ( From 1f801cb4bf407bcddd8f13ab9e16b01628858c3f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 3 Mar 2023 11:13:24 +0100 Subject: [PATCH 07/19] Email templates UI --- .../is/email/EmailTemplateController.java | 41 ++++++++ .../dnet-is-application/src/app/app.module.ts | 9 +- .../src/app/common/is.model.ts | 7 ++ .../src/app/common/is.service.ts | 22 ++++- .../src/app/emails/email-dialog.html | 41 ++++++++ .../src/app/emails/emails.component.html | 43 +++++++- .../src/app/emails/emails.component.ts | 98 ++++++++++++++++++- .../vocabularies/vocabularies.component.ts | 2 - .../dnetlib/common/model/EmailTemplate.java | 11 +++ .../src/main/resources/sql/schema.sql | 7 +- ...EmailDispatcher.java => EmailService.java} | 29 +++++- .../manager/wf/notification/EmailSender.java | 6 +- 12 files changed, 294 insertions(+), 22 deletions(-) create mode 100644 apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java create mode 100644 frontends/dnet-is-application/src/app/emails/email-dialog.html rename libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/{EmailDispatcher.java => EmailService.java} (84%) diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java new file mode 100644 index 00000000..425e9ce5 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java @@ -0,0 +1,41 @@ +package eu.dnetlib.is.email; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.common.model.EmailTemplate; +import eu.dnetlib.notifications.mail.EmailService; + +@RestController +@RequestMapping("/ajax/templates/email") +public class EmailTemplateController extends AbstractDnetController { + + @Autowired + protected EmailService emailService; + + @GetMapping("/") + public List listEmailTemplates() { + return emailService.listEmailTemplates(); + } + + @PostMapping("/") + public List saveEmailTemplate(@RequestBody final EmailTemplate email) { + emailService.saveEmailTemplate(email); + return emailService.listEmailTemplates(); + } + + @DeleteMapping("/{id}") + public List deleteEmailTemplate(@PathVariable final String id) { + emailService.deleteEmailTemplate(id); + return emailService.listEmailTemplates(); + } +} diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 22bea7a2..85955647 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -36,7 +36,7 @@ import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { SpinnerHttpInterceptor } from './common/spinner.service'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; -import { EmailsComponent } from './emails/emails.component'; +import { EmailDialog, EmailsComponent } from './emails/emails.component'; import { WfsComponent } from './wfs/wfs.component'; @NgModule({ @@ -68,9 +68,10 @@ import { WfsComponent } from './wfs/wfs.component'; MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog, - CleanerTesterComponent, - EmailsComponent, - WfsComponent + CleanerTesterComponent, + EmailsComponent, + EmailDialog, + WfsComponent ], imports: [ BrowserModule, diff --git a/frontends/dnet-is-application/src/app/common/is.model.ts b/frontends/dnet-is-application/src/app/common/is.model.ts index b6de0432..1da787c2 100644 --- a/frontends/dnet-is-application/src/app/common/is.model.ts +++ b/frontends/dnet-is-application/src/app/common/is.model.ts @@ -198,3 +198,10 @@ export interface MDStoreRecord { dateOfTransformation: string, provenance: any } + +export interface EmailTemplate { + id: string, + description: string, + subject: string, + message: string +} diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index ecef739e..b0dc096a 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; -import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model'; +import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord, EmailTemplate } from './is.model'; import { FormGroup } from '@angular/forms'; import { MatSnackBar } from '@angular/material/snack-bar'; @@ -328,6 +328,26 @@ export class ISService { }); } + loadEmailTemplates(onSuccess: Function): void { + this.client.get('./ajax/templates/email/').subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + saveEmailTemplate(email: EmailTemplate, onSuccess: Function, relatedForm?: FormGroup): void { + this.client.post('./ajax/templates/email/', email).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error, relatedForm) + }); + } + + deleteEmailTemplate(id: string, onSuccess: Function): void { + this.client.delete('./ajax/templates/email/' + encodeURIComponent(id)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } private showError(error: any, form?: FormGroup) { diff --git a/frontends/dnet-is-application/src/app/emails/email-dialog.html b/frontends/dnet-is-application/src/app/emails/email-dialog.html new file mode 100644 index 00000000..4ad7e7c0 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/email-dialog.html @@ -0,0 +1,41 @@ +
+ +

Edit Email Template

+

New Email Template

+ +
+ + + ID + + + + + Description + + This field is required + + + + Email: subject + + This field is required + + + + Email: message + + This field is required + + +
+ +
+ + + + {{ emailForm.errors?.['serverError'] }} + +
+ +
diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.html b/frontends/dnet-is-application/src/app/emails/emails.component.html index 57e79d74..f3b72955 100644 --- a/frontends/dnet-is-application/src/app/emails/emails.component.html +++ b/frontends/dnet-is-application/src/app/emails/emails.component.html @@ -1 +1,42 @@ -

emails works!

+

Email Templates

+ + + + + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Id + + {{element.id}} + Description {{element.description}} + +
No data matching the filter "{{input.value}}"
diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.ts b/frontends/dnet-is-application/src/app/emails/emails.component.ts index 166974f6..6a47eae7 100644 --- a/frontends/dnet-is-application/src/app/emails/emails.component.ts +++ b/frontends/dnet-is-application/src/app/emails/emails.component.ts @@ -1,10 +1,98 @@ -import { Component } from '@angular/core'; +import { AfterViewInit, Component, Inject, OnInit, ViewChild } from '@angular/core'; +import { FormControl, FormGroup, Validators } from '@angular/forms'; +import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MatSort } from '@angular/material/sort'; +import { MatTableDataSource } from '@angular/material/table'; +import { ActivatedRoute } from '@angular/router'; +import { EmailTemplate } from '../common/is.model'; +import { ISService } from '../common/is.service'; @Component({ - selector: 'app-emails', - templateUrl: './emails.component.html', - styleUrls: ['./emails.component.css'] + selector: 'app-emails', + templateUrl: './emails.component.html', + styleUrls: ['./emails.component.css'] }) -export class EmailsComponent { +export class EmailsComponent implements OnInit, AfterViewInit { + emailsDatasource: MatTableDataSource = new MatTableDataSource([]); + colums: string[] = ['id', 'description', 'buttons']; + + @ViewChild(MatSort) sort: MatSort | undefined + + searchText: string = ''; + + constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) { } + + ngOnInit() { this.reload() } + ngAfterViewInit() { if (this.sort) this.emailsDatasource.sort = this.sort; } + reload() { this.service.loadEmailTemplates((data: EmailTemplate[]) => this.emailsDatasource.data = data); } + + applyFilter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value.trim().toLowerCase(); + this.emailsDatasource.filter = filterValue; + } + + openAddEmailTemplateDialog(): void { + const dialogRef = this.dialog.open(EmailDialog, { + data: { + id: '', + description: '', + subject: '', + message: '' + }, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + openEditEmailTemplateDialog(email: EmailTemplate): void { + const dialogRef = this.dialog.open(EmailDialog, { + data: email, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + deleteEmailTemplate(email: EmailTemplate) { + if (confirm('Are you sure?')) { + this.service.deleteEmailTemplate(email.id, (data: void) => this.reload()); + } + } +} + +@Component({ + selector: 'email-dialog', + templateUrl: './email-dialog.html', + styleUrls: ['./emails.component.css'] +}) +export class EmailDialog { + + emailForm = new FormGroup({ + id: new FormControl(''), + description: new FormControl('', [Validators.required]), + subject: new FormControl('', [Validators.required]), + message: new FormControl('', [Validators.required]) + }); + + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { + this.emailForm.get('id')?.setValue(data.id); + this.emailForm.get('description')?.setValue(data.description); + this.emailForm.get('subject')?.setValue(data.subject); + this.emailForm.get('message')?.setValue(data.message); + } + + onSubmit(): void { + const email = Object.assign({}, this.data, this.emailForm.value); + this.service.saveEmailTemplate(email, (data: void) => this.dialogRef.close(1), this.emailForm); + } + + onNoClick(): void { + this.dialogRef.close(); + } } diff --git a/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts b/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts index 0d37f421..755f8f18 100644 --- a/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts +++ b/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts @@ -137,8 +137,6 @@ export class VocabularyEditorComponent implements OnInit, AfterViewInit { } } - - } @Component({ diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java index bd1a2849..b5369e4d 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java @@ -17,6 +17,9 @@ public class EmailTemplate implements Serializable { @Column(name = "id") private String id; + @Column(name = "description") + private String description; + @Column(name = "subject") private String subject; @@ -31,6 +34,14 @@ public class EmailTemplate implements Serializable { this.id = id; } + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + public String getSubject() { return subject; } diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index 18a60177..bf24c389 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -252,9 +252,10 @@ GROUP BY md.id, -- Email Templates CREATE TABLE emails ( - id text PRIMARY KEY, - subject text NOT NULL, - message text NOT NULL + id text PRIMARY KEY, + description text NOT NULL, + subject text NOT NULL, + message text NOT NULL ); -- Workflows diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java similarity index 84% rename from libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java rename to libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java index 0c6a2756..176b2b0d 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailDispatcher.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java @@ -1,9 +1,11 @@ package eu.dnetlib.notifications.mail; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -17,6 +19,7 @@ import javax.mail.Transport; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -25,12 +28,13 @@ import org.springframework.stereotype.Service; import com.google.common.base.Splitter; +import eu.dnetlib.common.model.EmailTemplate; import eu.dnetlib.common.repository.EmailTemplateRepository; @Service -public class EmailDispatcher { +public class EmailService { - private static final Log log = LogFactory.getLog(EmailDispatcher.class); + private static final Log log = LogFactory.getLog(EmailService.class); private final BlockingQueue queue = new LinkedBlockingQueue<>(); @Value("${dnet.configuration.mail.sender.email}") @@ -78,6 +82,22 @@ public class EmailDispatcher { }).start(); } + public List listEmailTemplates() { + return emailTemplateRepository.findAll(); + } + + public void saveEmailTemplate(final EmailTemplate email) { + if (StringUtils.isBlank(email.getId()) || email.getId().length() < 10) { + email.setId("email-" + UUID.randomUUID()); + log.info("Saving new email with id: " + email.getId()); + } + emailTemplateRepository.save(email); + } + + public void deleteEmailTemplate(final String id) { + emailTemplateRepository.deleteById(id); + } + public void sendMail(final String to, final String subject, final String message) { try { final Session session = Session.getInstance(obtainProperties(), obtainAuthenticator()); @@ -132,9 +152,12 @@ public class EmailDispatcher { private Authenticator obtainAuthenticator() { if (this.smtpUser == null || this.smtpUser.isEmpty()) { return null; } + final String user = this.smtpUser; + final String passwd = this.smtpPassword; + return new Authenticator() { - private final PasswordAuthentication authentication = new PasswordAuthentication(EmailDispatcher.this.smtpUser, EmailDispatcher.this.smtpPassword); + private final PasswordAuthentication authentication = new PasswordAuthentication(user, passwd); @Override protected PasswordAuthentication getPasswordAuthentication() { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java index e844d6d1..0f9d057a 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java @@ -12,7 +12,7 @@ import eu.dnetlib.manager.wf.model.NotificationCondition; import eu.dnetlib.manager.wf.repository.WorkflowSubscriptionRepository; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status; -import eu.dnetlib.notifications.mail.EmailDispatcher; +import eu.dnetlib.notifications.mail.EmailService; @Service public class EmailSender { @@ -23,7 +23,7 @@ public class EmailSender { private WorkflowSubscriptionRepository wfSubscriptionRepository; @Autowired - private EmailDispatcher dispatcher; + private EmailService emailService; public void sendMails(final WorkflowProcess proc) { @@ -34,7 +34,7 @@ public class EmailSender { try { final Map params = new HashMap<>(); - dispatcher.sendStoredMail(s.getEmail(), s.getMessageId(), params); + emailService.sendStoredMail(s.getEmail(), s.getMessageId(), params); } catch (final Exception e) { log.error("Error sending mail to " + s.getEmail(), e); From 1a6cc4ae2acf4249962099c2072e2a562b41ecbf Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 3 Mar 2023 13:53:21 +0100 Subject: [PATCH 08/19] add subtype to resources --- .../is/resource/ResourceAjaxController.java | 3 ++- .../src/app/common/is.model.ts | 1 + .../src/app/common/is.service.ts | 3 ++- .../src/app/resources/metadata-dialog.html | 7 ++++++- .../src/app/resources/new-dialog.html | 9 +++++++-- .../src/app/resources/resources.component.html | 12 ++++++++---- .../src/app/resources/resources.component.ts | 16 +++++++++++----- .../is/model/resource/SimpleResource.java | 11 +++++++++++ .../src/main/resources/sql/schema.sql | 1 + .../is/resource/SimpleResourceService.java | 2 ++ 10 files changed, 51 insertions(+), 14 deletions(-) diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java index e154e2f8..a9ec2707 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java @@ -25,11 +25,12 @@ public class ResourceAjaxController extends AbstractResourceController { @PostMapping("/") public SimpleResource newResource(@RequestParam final String name, @RequestParam final String type, + @RequestParam(required = false, defaultValue = "") final String subtype, @RequestParam(required = false, defaultValue = "") final String description, @RequestParam final String content) throws InformationServiceException { - return service.saveNewResource(name, type, description, content); + return service.saveNewResource(name, type, subtype, description, content); } @DeleteMapping("/{resId}") diff --git a/frontends/dnet-is-application/src/app/common/is.model.ts b/frontends/dnet-is-application/src/app/common/is.model.ts index 1da787c2..7ea47b6f 100644 --- a/frontends/dnet-is-application/src/app/common/is.model.ts +++ b/frontends/dnet-is-application/src/app/common/is.model.ts @@ -54,6 +54,7 @@ export interface SimpleResource { id: string, name: string, type: string, + subtype?: string, description?: string, creationDate?: string, modificationDate?: string diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index b0dc096a..ed6297dc 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -72,11 +72,12 @@ export class ISService { }); } - addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { + addSimpleResource(name: string, type: string, subtype: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') let body = new HttpParams() .set('name', name) .set('type', type) + .set('subtype', subtype) .set('description', description) .set('content', content); this.client.post('/ajax/resources/', body, { headers: headers }).subscribe({ diff --git a/frontends/dnet-is-application/src/app/resources/metadata-dialog.html b/frontends/dnet-is-application/src/app/resources/metadata-dialog.html index 1c391d15..c017f64d 100644 --- a/frontends/dnet-is-application/src/app/resources/metadata-dialog.html +++ b/frontends/dnet-is-application/src/app/resources/metadata-dialog.html @@ -13,9 +13,14 @@ + + SubType (optional) + + + Name - + This field is required diff --git a/frontends/dnet-is-application/src/app/resources/new-dialog.html b/frontends/dnet-is-application/src/app/resources/new-dialog.html index 4ed97939..031a4380 100644 --- a/frontends/dnet-is-application/src/app/resources/new-dialog.html +++ b/frontends/dnet-is-application/src/app/resources/new-dialog.html @@ -5,10 +5,15 @@ Name - + This field is required + + SubType (optional) + + + Description @@ -16,7 +21,7 @@ Content ({{data.contentType}}) - + This field is required diff --git a/frontends/dnet-is-application/src/app/resources/resources.component.html b/frontends/dnet-is-application/src/app/resources/resources.component.html index 17ce536b..6463697a 100644 --- a/frontends/dnet-is-application/src/app/resources/resources.component.html +++ b/frontends/dnet-is-application/src/app/resources/resources.component.html @@ -12,14 +12,18 @@ - {{r.name}} {{type.contentType}} + + {{r.name}} + {{r.subtype}} + {{type.contentType}} +

{{r.description}}

- Id: {{r.id}}
Creation date: {{r.creationDate}}
Modification date: - {{r.modificationDate}} + Id: {{r.id}}
+ Creation date: {{r.creationDate}}
+ Modification date: {{r.modificationDate}}

diff --git a/frontends/dnet-is-application/src/app/resources/resources.component.ts b/frontends/dnet-is-application/src/app/resources/resources.component.ts index c801a65a..1d5c2570 100644 --- a/frontends/dnet-is-application/src/app/resources/resources.component.ts +++ b/frontends/dnet-is-application/src/app/resources/resources.component.ts @@ -3,7 +3,7 @@ import { ISService } from '../common/is.service'; import { ActivatedRoute } from '@angular/router'; import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { ResourceType, SimpleResource } from '../common/is.model'; -import { FormControl, FormGroup } from '@angular/forms'; +import { FormControl, FormGroup, Validators } from '@angular/forms'; @Component({ selector: 'app-resources', @@ -115,12 +115,16 @@ export class ResContentDialog { }) export class ResMetadataDialog { metadataForm = new FormGroup({ - name: new FormControl(''), + name: new FormControl('', [Validators.required]), + subtype: new FormControl(''), description: new FormControl('') }); constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { this.metadataForm.get('name')?.setValue(data.name); + if (data.subtype) { + this.metadataForm.get('subtype')?.setValue(data.subtype); + } if (data.description) { this.metadataForm.get('description')?.setValue(data.description); } @@ -143,9 +147,10 @@ export class ResMetadataDialog { }) export class ResCreateNewDialog { newResourceForm = new FormGroup({ - name: new FormControl(''), + name: new FormControl('', [Validators.required]), + subtype: new FormControl(''), description: new FormControl(''), - content: new FormControl('') + content: new FormControl('', [Validators.required]) }); constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { } @@ -153,10 +158,11 @@ export class ResCreateNewDialog { onSubmit(): void { let name: string = this.newResourceForm.get('name')?.value!; let type: string = this.data.id!; + let subtype: string = this.newResourceForm.get('subtype')?.value!; let description: string = this.newResourceForm.get('description')?.value!; let content: string = this.newResourceForm.get('content')?.value!; - this.service.addSimpleResource(name, type, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm); + this.service.addSimpleResource(name, type, subtype, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm); } onNoClick(): void { this.dialogRef.close(); diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java index b585512f..6a73a64e 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java @@ -24,6 +24,9 @@ public class SimpleResource implements Serializable { @Column(name = "type") private String type; + @Column(name = "subtype") + private String subtype; + @Column(name = "description") private String description; @@ -57,6 +60,14 @@ public class SimpleResource implements Serializable { this.type = type; } + public String getSubtype() { + return subtype; + } + + public void setSubtype(final String subtype) { + this.subtype = subtype; + } + public String getDescription() { return description; } diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index bf24c389..58989441 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -139,6 +139,7 @@ CREATE TABLE resources ( description text, content text NOT NULL DEFAULT '', type text NOT NULL REFERENCES resource_types(id), + subtype text, creation_date timestamp NOT NULL DEFAULT now(), modification_date timestamp NOT NULL DEFAULT now() ); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java index 42fbb7cb..993c9fed 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java @@ -60,6 +60,7 @@ public class SimpleResourceService { @Transactional public SimpleResource saveNewResource(final String name, final String type, + final String subtype, final String description, final String content) throws InformationServiceException { @@ -71,6 +72,7 @@ public class SimpleResourceService { res.setId(UUID.randomUUID().toString()); res.setName(name); res.setType(type); + res.setSubtype(subtype); res.setDescription(description); res.setCreationDate(now); res.setModificationDate(now); From 0bd5e32eedb561118ac2244ef87d323d3de5f160 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 3 Mar 2023 15:22:46 +0100 Subject: [PATCH 09/19] refactoring --- .../{GraphArcDbEntry.java => GraphArc.java} | 2 +- .../{GraphNodeDbEntry.java => GraphNode.java} | 15 ++-- .../manager/wf/model/GraphParameter.java | 39 +++++++++ .../wf/model/GraphParameterDbEntry.java | 39 --------- ...eDbEntry.java => GraphParameterValue.java} | 2 +- .../manager/wf/model/WorkflowDbEntry.java | 83 ------------------- .../src/main/resources/sql/schema.sql | 19 ++--- .../repository/WorkflowDbEntryRepository.java | 9 -- .../wf/workflows/graph/GraphLoader.java | 20 ++--- .../wf/workflows/procs/ProcessFactory.java | 17 ++-- .../wf/workflows/procs/WorkflowExecutor.java | 40 +++++++-- 11 files changed, 108 insertions(+), 177 deletions(-) rename libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/{GraphArcDbEntry.java => GraphArc.java} (87%) rename libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/{GraphNodeDbEntry.java => GraphNode.java} (71%) create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java rename libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/{GraphParameterValueDbEntry.java => GraphParameterValue.java} (91%) delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java delete mode 100644 libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java similarity index 87% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java index 5ec1447f..b5f4ac58 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArcDbEntry.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java @@ -2,7 +2,7 @@ package eu.dnetlib.manager.wf.model; import java.io.Serializable; -public class GraphArcDbEntry implements Serializable { +public class GraphArc implements Serializable { private static final long serialVersionUID = 7866138976929522262L; diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java similarity index 71% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java index ca1e0f91..c506c8f1 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNodeDbEntry.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java @@ -1,8 +1,9 @@ package eu.dnetlib.manager.wf.model; import java.io.Serializable; +import java.util.List; -public class GraphNodeDbEntry implements Serializable { +public class GraphNode implements Serializable { private static final long serialVersionUID = -3695762832959801906L; @@ -10,8 +11,8 @@ public class GraphNodeDbEntry implements Serializable { private String type; private boolean isStart = false; private boolean isJoin = false; - private GraphArcDbEntry[] arcs; - private GraphParameterDbEntry[] params; + private List arcs; + private List params; public String getName() { return name; @@ -45,19 +46,19 @@ public class GraphNodeDbEntry implements Serializable { this.isJoin = isJoin; } - public GraphArcDbEntry[] getArcs() { + public List getArcs() { return arcs; } - public void setArcs(final GraphArcDbEntry[] arcs) { + public void setArcs(final List arcs) { this.arcs = arcs; } - public GraphParameterDbEntry[] getParams() { + public List getParams() { return params; } - public void setParams(final GraphParameterDbEntry[] params) { + public void setParams(final List params) { this.params = params; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java new file mode 100644 index 00000000..f24b12c4 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java @@ -0,0 +1,39 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +public class GraphParameter extends GraphParameterValue implements Serializable { + + private static final long serialVersionUID = 1894419948433994453L; + + private String name; + private List values; + private Map map; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public List getValues() { + return values; + } + + public void setValues(final List values) { + this.values = values; + } + + public Map getMap() { + return map; + } + + public void setMap(final Map map) { + this.map = map; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java deleted file mode 100644 index 6edcf0f7..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterDbEntry.java +++ /dev/null @@ -1,39 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -public class GraphParameterDbEntry extends GraphParameterValueDbEntry implements Serializable { - - private static final long serialVersionUID = 1894419948433994453L; - - private String name; - private List values; - private Map map; - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public List getValues() { - return values; - } - - public void setValues(final List values) { - this.values = values; - } - - public Map getMap() { - return map; - } - - public void setMap(final Map map) { - this.map = map; - } - -} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java similarity index 91% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java index c04a6e30..ee8267f9 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValueDbEntry.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java @@ -2,7 +2,7 @@ package eu.dnetlib.manager.wf.model; import java.io.Serializable; -public class GraphParameterValueDbEntry implements Serializable { +public class GraphParameterValue implements Serializable { private static final long serialVersionUID = 7815785723401725707L; diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java deleted file mode 100644 index f991bec7..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowDbEntry.java +++ /dev/null @@ -1,83 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; - -import org.hibernate.annotations.Type; -import org.hibernate.annotations.TypeDef; -import org.hibernate.annotations.TypeDefs; - -import com.vladmihalcea.hibernate.type.json.JsonBinaryType; -import com.vladmihalcea.hibernate.type.json.JsonStringType; - -@Entity -@Table(name = "workflows") -@TypeDefs({ - @TypeDef(name = "json", typeClass = JsonStringType.class), - @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) -}) -public class WorkflowDbEntry implements Serializable { - - private static final long serialVersionUID = -4684952453322136556L; - - @Id - @Column(name = "id") - private String id; - - @Column(name = "name") - private String name; - - @Column(name = "family") - private String family; - - @Column(name = "description") - private String description; - - @Type(type = "jsonb") - @Column(name = "data", columnDefinition = "jsonb") - private GraphNodeDbEntry[] graph; - - public String getId() { - return id; - } - - public void setId(final String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public String getFamily() { - return family; - } - - public void setFamily(final String family) { - this.family = family; - } - - public String getDescription() { - return description; - } - - public void setDescription(final String description) { - this.description = description; - } - - public GraphNodeDbEntry[] getGraph() { - return graph; - } - - public void setGraph(final GraphNodeDbEntry[] graph) { - this.graph = graph; - } -} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index 58989441..a4f66ba8 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -131,7 +131,8 @@ INSERT INTO resource_types(id, name, content_type) VALUES ('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'), ('cleaning_rule', 'Cleaning Rules', 'application/xml'), ('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml') - ('dedup_configuration', 'Dedup Configurations', 'application/json'); + ('dedup_configuration', 'Dedup Configurations', 'application/json') + ('workflow', 'Workflows', 'application/json'); CREATE TABLE resources ( id text PRIMARY KEY, @@ -261,14 +262,6 @@ CREATE TABLE emails ( -- Workflows -CREATE TABLE workflows ( - id text PRIMARY KEY, - name text NOT NULL, - family text NOT NULL, - description text, - data jsonb NOT NULL -); - CREATE TABLE workflow_instances ( id text PRIMARY KEY, details jsonb NOT NULL DEFAULT '{}', @@ -281,14 +274,15 @@ CREATE TABLE workflow_instances ( scheduling_enabled boolean NOT NULL DEFAULT false, scheduling_cron text, scheduling_min_interval int, - workflow text REFERENCES workflows(id), - destroy_wf text REFERENCES workflows(id), + workflow text REFERENCES resource(id), + destroy_wf text REFERENCES resource(id), system_params jsonb NOT NULL DEFAULT '{}', user_params jsonb NOT NULL DEFAULT '{}' ); +-- TO DELETE CREATE TABLE workflow_expected_params ( - wf_id text REFERENCES workflows(id), + wf_id text REFERENCES resource(id), name text, description text, type text, @@ -296,6 +290,7 @@ CREATE TABLE workflow_expected_params ( default_value text, PRIMARY KEY (wf_id, name) ); +-- END CREATE TABLE workflow_subscriptions ( wf_instance_id text NOT NULL REFERENCES workflow_instances(id), diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java deleted file mode 100644 index 9e007e54..00000000 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowDbEntryRepository.java +++ /dev/null @@ -1,9 +0,0 @@ -package eu.dnetlib.manager.wf.repository; - -import org.springframework.data.jpa.repository.JpaRepository; - -import eu.dnetlib.manager.wf.model.WorkflowDbEntry; - -public interface WorkflowDbEntryRepository extends JpaRepository { - -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java index 18f5d2d6..3a42b8e4 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -19,10 +19,9 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.model.GraphArcDbEntry; -import eu.dnetlib.manager.wf.model.GraphNodeDbEntry; -import eu.dnetlib.manager.wf.model.GraphParameterDbEntry; -import eu.dnetlib.manager.wf.model.GraphParameterValueDbEntry; +import eu.dnetlib.manager.wf.model.GraphArc; +import eu.dnetlib.manager.wf.model.GraphParameter; +import eu.dnetlib.manager.wf.model.GraphParameterValue; import eu.dnetlib.manager.wf.workflows.util.NodeHelper; @Service @@ -40,10 +39,11 @@ public class GraphLoader { @Autowired private Environment env; - public Graph loadGraph(final GraphNodeDbEntry[] workflowGraph, final Map globalParams) throws WorkflowManagerException { + public Graph loadGraph(final List workflowGraph, final Map globalParams) + throws WorkflowManagerException { final Graph graph = new Graph(); - for (final GraphNodeDbEntry node : workflowGraph) { + for (final eu.dnetlib.manager.wf.model.GraphNode node : workflowGraph) { final String nodeName = node.getName(); final String nodeType = node.getType(); final boolean isStart = node.isStart(); @@ -60,7 +60,7 @@ public class GraphLoader { } if (graph.getArcs() != null) { - for (final GraphArcDbEntry a : node.getArcs()) { + for (final GraphArc a : node.getArcs()) { final String arcName = a.getName(); final String to = a.getTo(); graph.addArc(new Arc(StringUtils.isNotBlank(arcName) ? arcName : Arc.DEFAULT_ARC, nodeName, to)); @@ -75,12 +75,12 @@ public class GraphLoader { return graph; } - public Map calculateParamsForNode(final GraphNodeDbEntry node, final Map globalParams) { + public Map calculateParamsForNode(final eu.dnetlib.manager.wf.model.GraphNode node, final Map globalParams) { final Map params = new HashMap<>(); if (node.getParams() != null) { - for (final GraphParameterDbEntry p : node.getParams()) { + for (final GraphParameter p : node.getParams()) { final String pName = p.getName(); @@ -116,7 +116,7 @@ public class GraphLoader { return params; } - private GraphNodeParameter calculateSimpleValue(final GraphParameterValueDbEntry graphValue, final Map globalParams) { + private GraphNodeParameter calculateSimpleValue(final GraphParameterValue graphValue, final Map globalParams) { String value = graphValue.getValue(); final String ref = graphValue.getRef(); final String prop = graphValue.getProperty(); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index 8cf041e3..2aae0a94 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -3,6 +3,7 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -11,7 +12,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.model.WorkflowDbEntry; +import eu.dnetlib.manager.wf.model.GraphNode; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; @@ -28,7 +29,10 @@ public class ProcessFactory { @Autowired private GraphLoader graphLoader; - public WorkflowProcess newProcess(final WorkflowDbEntry wf, + public WorkflowProcess newProcess(final String wfId, + final String wfName, + final String wfFamily, + final List graph, final WorkflowInstance instance, final ProcessCallback processCallback, final String parent) throws WorkflowManagerException { @@ -38,17 +42,16 @@ public class ProcessFactory { globalParams.putAll(instance.getUserParams()); return new WorkflowProcess(generateProcessId(), - wf.getName(), - wf.getFamily(), + wfName, + wfFamily, instance.getDsId(), instance.getDsName(), instance.getApiId(), - graphLoader.loadGraph(wf.getGraph(), globalParams), + graphLoader.loadGraph(graph, globalParams), instance.getPriority(), - wf.getId(), + wfId, instance.getId(), globalParams, - processCallback, parent); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java index 44ea7223..cee2a1c7 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -1,6 +1,8 @@ package eu.dnetlib.manager.wf.workflows.procs; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -12,12 +14,16 @@ import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + import eu.dnetlib.dsm.DsmService; import eu.dnetlib.errors.DsmException; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.model.WorkflowDbEntry; +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.is.resource.repository.SimpleResourceRepository; +import eu.dnetlib.manager.wf.model.GraphNode; import eu.dnetlib.manager.wf.model.WorkflowInstance; -import eu.dnetlib.manager.wf.repository.WorkflowDbEntryRepository; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; @@ -39,7 +45,7 @@ public class WorkflowExecutor implements Stoppable { private DsmService dsmService; @Autowired - private WorkflowDbEntryRepository workflowDbEntryRepository; + private SimpleResourceRepository simpleResourceRepository; @Autowired private WorkflowInstanceRepository workflowInstanceRepository; @@ -107,16 +113,34 @@ public class WorkflowExecutor implements Stoppable { public String startWorkflowInstance(final WorkflowInstance instance, final ProcessCallback processCallback, final String parent) throws WorkflowManagerException { - final WorkflowDbEntry wf = workflowDbEntryRepository.findById(instance.getWorkflow()) - .orElseThrow(() -> new WorkflowManagerException("WF not found: " + instance.getWorkflow())); if (!instance.isEnabled() || !instance.isConfigured()) { - log.warn("Wf instance " + instance.getId() + " not launched, because it is not ready to start or it is disabled"); - throw new WorkflowManagerException("Workflow " + instance.getId() + " is not ready to start"); + log.warn("Wf instance " + instance.getId() + " is not ready to start"); + throw new WorkflowManagerException("Wf instance " + instance.getId() + " is not ready to start"); } + final SimpleResource wf = simpleResourceRepository + .findById(instance.getWorkflow()) + .filter(r -> r.getType().equals("workflows")) + .orElseThrow(() -> new WorkflowManagerException("WF not found: " + instance.getWorkflow())); + + final List graph = simpleResourceRepository.findContentById(wf.getId()) + .map(s -> { + final ObjectMapper mapper = new ObjectMapper(); + final TypeReference> type = new TypeReference>() {}; + final List list = new ArrayList<>(); + try { + list.addAll(mapper.readValue(s, type)); + } catch (final Exception e) { + log.error("Error parsing json", e); + } + return list; + }) + .filter(list -> !list.isEmpty()) + .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wf.getId())); + final WorkflowProcess process = - processFactory.newProcess(wf, instance, processCallback, parent); + processFactory.newProcess(wf.getId(), wf.getName(), wf.getSubtype(), graph, instance, processCallback, parent); return processRegistry.registerProcess(process, instance.getId()); } From e4443ef2c8a7b85232f7ee5d3f2839a9e4bb6bd5 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 6 Mar 2023 14:04:01 +0100 Subject: [PATCH 10/19] refactoring --- .../eu/dnetlib/manager/wf/model/GraphArc.java | 28 -- .../dnetlib/manager/wf/model/GraphNode.java | 65 ----- .../manager/wf/model/GraphParameter.java | 39 --- .../manager/wf/model/GraphParameterValue.java | 45 --- .../manager/wf/model/WorkflowGraph.java | 270 ++++++++++++++++++ .../src/main/resources/sql/schema.sql | 12 - .../wf/cron/ScheduledWorkflowLauncher.java | 5 +- .../wf/workflows/graph/GraphLoader.java | 105 +------ .../manager/wf/workflows/graph/GraphNode.java | 78 ++--- .../workflows/graph/GraphNodeParameter.java | 56 ---- .../wf/workflows/procs/ProcessFactory.java | 32 +-- .../wf/workflows/procs/ProcessRegistry.java | 40 ++- .../wf/workflows/procs/WorkflowExecutor.java | 38 ++- .../wf/workflows/procs/WorkflowProcess.java | 164 ++++------- 14 files changed, 417 insertions(+), 560 deletions(-) delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java delete mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java deleted file mode 100644 index b5f4ac58..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphArc.java +++ /dev/null @@ -1,28 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; - -public class GraphArc implements Serializable { - - private static final long serialVersionUID = 7866138976929522262L; - - private String name; - private String to; - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public String getTo() { - return to; - } - - public void setTo(final String to) { - this.to = to; - } - -} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java deleted file mode 100644 index c506c8f1..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphNode.java +++ /dev/null @@ -1,65 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; -import java.util.List; - -public class GraphNode implements Serializable { - - private static final long serialVersionUID = -3695762832959801906L; - - private String name; - private String type; - private boolean isStart = false; - private boolean isJoin = false; - private List arcs; - private List params; - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(final String type) { - this.type = type; - } - - public boolean isStart() { - return isStart; - } - - public void setStart(final boolean isStart) { - this.isStart = isStart; - } - - public boolean isJoin() { - return isJoin; - } - - public void setJoin(final boolean isJoin) { - this.isJoin = isJoin; - } - - public List getArcs() { - return arcs; - } - - public void setArcs(final List arcs) { - this.arcs = arcs; - } - - public List getParams() { - return params; - } - - public void setParams(final List params) { - this.params = params; - } - -} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java deleted file mode 100644 index f24b12c4..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameter.java +++ /dev/null @@ -1,39 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -public class GraphParameter extends GraphParameterValue implements Serializable { - - private static final long serialVersionUID = 1894419948433994453L; - - private String name; - private List values; - private Map map; - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } - - public List getValues() { - return values; - } - - public void setValues(final List values) { - this.values = values; - } - - public Map getMap() { - return map; - } - - public void setMap(final Map map) { - this.map = map; - } - -} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java deleted file mode 100644 index ee8267f9..00000000 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/GraphParameterValue.java +++ /dev/null @@ -1,45 +0,0 @@ -package eu.dnetlib.manager.wf.model; - -import java.io.Serializable; - -public class GraphParameterValue implements Serializable { - - private static final long serialVersionUID = 7815785723401725707L; - - private String value; - private String ref; - private String property; - private String env; - - public String getValue() { - return value; - } - - public void setValue(final String value) { - this.value = value; - } - - public String getRef() { - return ref; - } - - public void setRef(final String ref) { - this.ref = ref; - } - - public String getProperty() { - return property; - } - - public void setProperty(final String property) { - this.property = property; - } - - public String getEnv() { - return env; - } - - public void setEnv(final String env) { - this.env = env; - } -} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java new file mode 100644 index 00000000..eae06aa1 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java @@ -0,0 +1,270 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.env.Environment; + +public class WorkflowGraph implements Serializable { + + private static final long serialVersionUID = 5919290887480115842L; + + public List parameters; + public List graph; + + public List getGraph() { + return graph; + } + + public void setGraph(final List graph) { + this.graph = graph; + } + + public List getParameters() { + return parameters; + } + + public void setParameters(final List parameters) { + this.parameters = parameters; + } + + class WfParam implements Serializable { + + private static final long serialVersionUID = 5885589803738655166L; + + private String name; + private String description; + private String type; + private String defaultValue; + private boolean required; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(final String defaultValue) { + this.defaultValue = defaultValue; + } + + public boolean isRequired() { + return required; + } + + public void setRequired(final boolean required) { + this.required = required; + } + + } + + public class Node implements Serializable { + + private static final long serialVersionUID = -3695762832959801906L; + + private static final String regExRef = "\\$\\{(\\w*)\\}"; + + private String name; + private String type; + private boolean isStart = false; + private boolean isJoin = false; + private List arcs; + private List input; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public boolean isStart() { + return isStart; + } + + public void setStart(final boolean isStart) { + this.isStart = isStart; + } + + public boolean isJoin() { + return isJoin; + } + + public void setJoin(final boolean isJoin) { + this.isJoin = isJoin; + } + + public List getArcs() { + return arcs; + } + + public void setArcs(final List arcs) { + this.arcs = arcs; + } + + public List getInput() { + return input; + } + + public void setInput(final List input) { + this.input = input; + } + + public Map findEnvParams() { + return input.stream() + .filter(p -> StringUtils.isNotBlank(p.getEnv())) + .collect(Collectors.toMap(NodeParam::getName, NodeParam::getEnv)); + } + + public Map calculateInitialParams(final Map globalParams, final Environment environment) { + final Map map = new HashMap<>(); + + input.stream() + .filter(p -> StringUtils.isBlank(p.getEnv())) + .forEach(p -> map.put(p.getName(), calculateSimpleValue(p, globalParams, environment))); + + return map; + } + + private Object calculateSimpleValue(final NodeParam p, final Map globalParams, final Environment environment) { + String value = p.getValue(); + final String ref = p.getRef(); + final String prop = p.getProperty(); + + if (StringUtils.isNotBlank(ref) && StringUtils.isNotBlank(globalParams.get(ref))) { + return globalParams.get(ref); + } else if (StringUtils.isNotBlank(value)) { + final Matcher matcher = Pattern.compile(regExRef, Pattern.MULTILINE).matcher(value); + while (matcher.find()) { + final String rName = matcher.group(1); + final String rValue = globalParams.get(rName); + if (StringUtils.isBlank(rValue)) { return null; } + value = value.replaceAll(Pattern.quote(matcher.group(0)), rValue); + System.out.println("NEW VALUE " + value); + } + return value; + } else if (StringUtils.isNotBlank(prop)) { + return environment.getProperty(prop); + } else { + return null; + } + + } + + } + + public class Arc implements Serializable { + + private static final long serialVersionUID = 7866138976929522262L; + + private String name; + private String to; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getTo() { + return to; + } + + public void setTo(final String to) { + this.to = to; + } + + } + + class NodeParam implements Serializable { + + private static final long serialVersionUID = 7815785723401725707L; + + private String name; + private String value; + private String ref; + private String property; + private String env; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + public String getRef() { + return ref; + } + + public void setRef(final String ref) { + this.ref = ref; + } + + public String getProperty() { + return property; + } + + public void setProperty(final String property) { + this.property = property; + } + + public String getEnv() { + return env; + } + + public void setEnv(final String env) { + this.env = env; + } + + } + +} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index a4f66ba8..2952dc26 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -280,18 +280,6 @@ CREATE TABLE workflow_instances ( user_params jsonb NOT NULL DEFAULT '{}' ); --- TO DELETE -CREATE TABLE workflow_expected_params ( - wf_id text REFERENCES resource(id), - name text, - description text, - type text, - required boolean, - default_value text, - PRIMARY KEY (wf_id, name) -); --- END - CREATE TABLE workflow_subscriptions ( wf_instance_id text NOT NULL REFERENCES workflow_instances(id), condition text NOT NULL, diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 3711a3e1..0eec4f58 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -122,7 +122,9 @@ public class ScheduledWorkflowLauncher { } private boolean isNotRunning(final WorkflowInstance instance) { - for (final WorkflowProcess p : processRegistry.findProcsByOtherId(instance.getId())) { + final WorkflowProcess p = processRegistry.findProcsByInstanceId(instance.getId()); + + if (p != null) { switch (p.getStatus()) { case CREATED: return false; @@ -132,6 +134,7 @@ public class ScheduledWorkflowLauncher { break; } } + return true; } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java index 3a42b8e4..6af1f94d 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -1,17 +1,10 @@ package eu.dnetlib.manager.wf.workflows.graph; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; @@ -19,48 +12,41 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.model.GraphArc; -import eu.dnetlib.manager.wf.model.GraphParameter; -import eu.dnetlib.manager.wf.model.GraphParameterValue; +import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.workflows.util.NodeHelper; @Service public class GraphLoader { - private static final Log log = LogFactory.getLog(GraphLoader.class); - - private final String regExRef = "\\$\\{(\\w*)\\}"; - - final Pattern pattern = Pattern.compile(regExRef, Pattern.MULTILINE); - @Autowired private NodeHelper nodeHelper; @Autowired - private Environment env; + private Environment environment; - public Graph loadGraph(final List workflowGraph, final Map globalParams) + public Graph loadGraph(final WorkflowGraph workflowGraph, final Map globalParams) throws WorkflowManagerException { final Graph graph = new Graph(); - for (final eu.dnetlib.manager.wf.model.GraphNode node : workflowGraph) { + for (final WorkflowGraph.Node node : workflowGraph.getGraph()) { final String nodeName = node.getName(); final String nodeType = node.getType(); final boolean isStart = node.isStart(); final boolean isJoin = node.isJoin(); - final Map params = calculateParamsForNode(node, globalParams); + final Map params = node.calculateInitialParams(globalParams, environment); + final Map envParams = node.findEnvParams(); if (isStart) { - graph.addNode(GraphNode.newStartNode(nodeName, nodeType, params)); + graph.addNode(GraphNode.newStartNode(nodeName, nodeType, params, envParams)); } else if (isJoin) { - graph.addNode(GraphNode.newJoinNode(nodeName, nodeType, params)); + graph.addNode(GraphNode.newJoinNode(nodeName, nodeType, params, envParams)); } else { - graph.addNode(GraphNode.newNode(nodeName, nodeType, params)); + graph.addNode(GraphNode.newNode(nodeName, nodeType, params, envParams)); } - if (graph.getArcs() != null) { - for (final GraphArc a : node.getArcs()) { + if (node.getArcs() != null) { + for (final WorkflowGraph.Arc a : node.getArcs()) { final String arcName = a.getName(); final String to = a.getTo(); graph.addArc(new Arc(StringUtils.isNotBlank(arcName) ? arcName : Arc.DEFAULT_ARC, nodeName, to)); @@ -75,75 +61,6 @@ public class GraphLoader { return graph; } - public Map calculateParamsForNode(final eu.dnetlib.manager.wf.model.GraphNode node, final Map globalParams) { - - final Map params = new HashMap<>(); - - if (node.getParams() != null) { - for (final GraphParameter p : node.getParams()) { - - final String pName = p.getName(); - - final GraphNodeParameter pValue = calculateSimpleValue(p, globalParams); - - if (pValue != null) { - params.put(pName, pValue); - } else if (p.getMap() != null) { - - final Map map = p.getMap() - .entrySet() - .stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> { - final GraphNodeParameter gnp = calculateSimpleValue(e.getValue(), globalParams); - if (gnp == null) { - final String msg = String.format("missing value for param: \"%s\"", e.getKey()); - log.debug(msg); - return GraphNodeParameter.newNullParam(); - } - return gnp; - })); - params.put(pName, GraphNodeParameter.newMapParam(map)); - } else if (p.getValues() != null) { - final List list = p.getValues() - .stream() - .map(e -> calculateSimpleValue(e, globalParams)) - .collect(Collectors.toList()); - params.put(pName, GraphNodeParameter.newListParam(list)); - } - } - } - - return params; - } - - private GraphNodeParameter calculateSimpleValue(final GraphParameterValue graphValue, final Map globalParams) { - String value = graphValue.getValue(); - final String ref = graphValue.getRef(); - final String prop = graphValue.getProperty(); - final String envRef = graphValue.getEnv(); - - if (StringUtils.isNotBlank(ref) && StringUtils.isNotBlank(globalParams.get(ref))) { - return GraphNodeParameter.newSimpleParam(globalParams.get(ref)); - } else if (StringUtils.isNotBlank(envRef)) { - return GraphNodeParameter.newEnvParam(envRef); - } else if (StringUtils.isNotBlank(value)) { - final Matcher matcher = pattern.matcher(value); - while (matcher.find()) { - final String rName = matcher.group(1); - final String rValue = globalParams.get(rName); - if (StringUtils.isBlank(rValue)) { return null; } - value = value.replaceAll(Pattern.quote(matcher.group(0)), rValue); - System.out.println("NEW VALUE " + value); - } - return GraphNodeParameter.newSimpleParam(value); - } else if (StringUtils.isNotBlank(prop)) { - return GraphNodeParameter.newSimpleParam(env.getProperty(prop)); - } else { - return null; - } - - } - private void checkValidity(final Graph graph) throws WorkflowManagerException { final Set nodesFromArcs = new HashSet<>(); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java index db9c5c67..9f3429e2 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java @@ -2,9 +2,7 @@ package eu.dnetlib.manager.wf.workflows.graph; import java.io.StringWriter; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -19,42 +17,48 @@ public class GraphNode { private final boolean isStart; private final boolean isJoin; private final boolean isSucessNode; - private final Map nodeParams; + private final Map params; + private final Map envParams; private GraphNode(final String name, final String type, final boolean isStart, final boolean isJoin, final boolean isSuccessNode, - final Map nodeParams) { + final Map params, + final Map envParams) { this.name = name; this.type = type; this.isStart = isStart; this.isJoin = isJoin; this.isSucessNode = isSuccessNode; - this.nodeParams = nodeParams; + this.params = params; + this.envParams = envParams; } public static GraphNode newNode(final String name, final String type, - final Map nodeParams) { - return new GraphNode(name, type, false, false, false, nodeParams); + final Map params, + final Map envParams) { + return new GraphNode(name, type, false, false, false, params, envParams); } public static GraphNode newStartNode(final String name, final String type, - final Map nodeParams) { - return new GraphNode(name, type, true, false, false, nodeParams); + final Map params, + final Map envParams) { + return new GraphNode(name, type, true, false, false, params, envParams); } public static GraphNode newJoinNode(final String name, final String type, - final Map nodeParams) { - return new GraphNode(name, type, false, true, false, nodeParams); + final Map params, + final Map envParams) { + return new GraphNode(name, type, false, true, false, params, envParams); } public static GraphNode newSuccessNode() { - return new GraphNode(SUCCESS_NODE, null, false, true, true, null); + return new GraphNode(SUCCESS_NODE, null, false, true, true, new HashMap<>(), new HashMap<>()); } public String getName() { @@ -96,54 +100,30 @@ public class GraphNode { return sw.toString(); } - public Map getNodeParams() { - return this.nodeParams; + public Map getParams() { + return this.params; + } + + public Map getEnvParams() { + return this.envParams; } public Map resolveParamsWithNoEnv() { return resolveParams(null); } - @SuppressWarnings("unchecked") public Map resolveParams(final Env env) { - final Map params = new HashMap<>(); + final Map map = new HashMap<>(); - if (this.nodeParams != null) { - - for (final Map.Entry e : this.nodeParams.entrySet()) { - final String pName = e.getKey(); - final GraphNodeParameter param = e.getValue(); - - if (param.isEnvParam()) { - params.put(pName, resolveFromEnv(param, env)); - } else if (param.isMap()) { - - final Map map = new HashMap<>(); - - for (final Map.Entry e1 : ((Map) param.getValue()).entrySet()) { - map.put(e1.getKey(), e1.getValue().isEnvParam() ? resolveFromEnv(e1.getValue(), env) : e1.getValue().getValue()); - } - - params.put(pName, map); - - } else if (param.isList()) { - params.put(pName, ((List) param.getValue()) - .stream() - .map(p -> p.isEnvParam() ? resolveFromEnv(p, env) : p.getValue()) - .collect(Collectors.toList())); - - } else { - params.put(pName, param.getValue()); - } - - } + if (this.params != null) { + this.params.forEach((k, v) -> map.put(k, v)); } - return params; - } + if (this.envParams != null && env != null) { + this.envParams.forEach((k, v) -> map.put(k, env.getAttribute(v))); + } - private Object resolveFromEnv(final GraphNodeParameter param, final Env env) { - return env != null ? env.getAttribute(param.getEnvVariable()) : "[this value will be resolved using the runtime ENV]"; + return map; } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java deleted file mode 100644 index bbe054eb..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNodeParameter.java +++ /dev/null @@ -1,56 +0,0 @@ -package eu.dnetlib.manager.wf.workflows.graph; - -import java.util.List; -import java.util.Map; - -public class GraphNodeParameter { - - private final Object value; - private final String envVariable; - - private GraphNodeParameter(final Object value, final String envVariable) { - this.value = value; - this.envVariable = envVariable; - } - - public static GraphNodeParameter newNullParam() { - return new GraphNodeParameter(null, null); - } - - public static GraphNodeParameter newSimpleParam(final Object value) { - return new GraphNodeParameter(value, null); - } - - public static GraphNodeParameter newMapParam(final Map map) { - return new GraphNodeParameter(map, null); - } - - public static GraphNodeParameter newListParam(final List list) { - return new GraphNodeParameter(list, null); - } - - public static GraphNodeParameter newEnvParam(final String envVariable) { - return new GraphNodeParameter(null, envVariable); - } - - public Object getValue() { - return this.value; - } - - public boolean isEnvParam() { - return this.envVariable != null; - } - - public String getEnvVariable() { - return this.envVariable; - } - - public boolean isMap() { - return this.value != null && (this.value instanceof Map); - } - - public boolean isList() { - return this.value != null && (this.value instanceof List); - } - -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index 2aae0a94..c7c07ace 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -3,7 +3,6 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -12,8 +11,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.model.GraphNode; +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.graph.Graph; import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; @@ -29,30 +30,19 @@ public class ProcessFactory { @Autowired private GraphLoader graphLoader; - public WorkflowProcess newProcess(final String wfId, - final String wfName, - final String wfFamily, - final List graph, - final WorkflowInstance instance, + public WorkflowProcess newProcess(final SimpleResource wfMetadata, + final WorkflowGraph wfGraph, + final WorkflowInstance wfInstance, final ProcessCallback processCallback, final String parent) throws WorkflowManagerException { final Map globalParams = new HashMap<>(); - globalParams.putAll(instance.getSystemParams()); - globalParams.putAll(instance.getUserParams()); + globalParams.putAll(wfInstance.getSystemParams()); + globalParams.putAll(wfInstance.getUserParams()); - return new WorkflowProcess(generateProcessId(), - wfName, - wfFamily, - instance.getDsId(), - instance.getDsName(), - instance.getApiId(), - graphLoader.loadGraph(graph, globalParams), - instance.getPriority(), - wfId, - instance.getId(), - globalParams, - processCallback, parent); + final Graph graph = graphLoader.loadGraph(wfGraph, globalParams); + + return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, processCallback, parent); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java index 45533072..fd1180b6 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java @@ -1,11 +1,10 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Set; +import java.util.Optional; import java.util.concurrent.PriorityBlockingQueue; import org.apache.commons.logging.Log; @@ -13,18 +12,16 @@ import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; - import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; @Service public class ProcessRegistry { private static final Log log = LogFactory.getLog(ProcessRegistry.class); - private final BiMap procs = HashBiMap.create(); - private final Map> byOtherId = new HashMap<>(); + private final Map procs = new HashMap<>(); + private final Map byInstanceId = new HashMap<>(); private final PriorityBlockingQueue pendingProcs = new PriorityBlockingQueue<>(); @@ -46,18 +43,15 @@ public class ProcessRegistry { return this.procs.get(procId); } - public Set listProcesses() { + public Collection listProcesses() { return this.procs.values(); } - public Collection findProcsByOtherId(final String id) { - synchronized (this) { - final Collection res = this.byOtherId.get(id); - return res != null ? res : new ArrayList<>(); - } + public WorkflowProcess findProcsByInstanceId(final String id) { + return this.byInstanceId.get(id); } - public String registerProcess(final WorkflowProcess process, final String... ids) throws WorkflowManagerException { + public String registerProcess(final WorkflowProcess process, final WorkflowInstance wfInstance) throws WorkflowManagerException { if (this.procs.containsValue(process) || this.procs.containsKey(process.getId())) { log.error("Already registerd process: " + process); throw new WorkflowManagerException("Already registerd process: " + process); @@ -68,14 +62,7 @@ public class ProcessRegistry { } this.procs.put(process.getId(), process); - for (final String id : ids) { - synchronized (this) { - if (!this.byOtherId.containsKey(id)) { - this.byOtherId.put(id, new ArrayList()); - } - this.byOtherId.get(id).add(process); - } - } + this.byInstanceId.put(wfInstance.getId(), process); synchronized (this.pendingProcs) { if (this.pendingProcs.size() > WorkflowsConstants.MAX_PENDING_PROCS_SIZE) { @@ -116,8 +103,13 @@ public class ProcessRegistry { synchronized (this) { final WorkflowProcess process = this.procs.remove(procId); if (process != null) { - for (final Collection processes : this.byOtherId.values()) { - processes.remove(process); + final Optional instanceId = this.byInstanceId.entrySet() + .stream() + .filter(e -> e.getValue().getId().equals(process.getId())) + .map(e -> e.getKey()) + .findFirst(); + if (instanceId.isPresent()) { + this.byInstanceId.remove(instanceId, process); } } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java index cee2a1c7..e83add35 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -1,8 +1,7 @@ package eu.dnetlib.manager.wf.workflows.procs; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -14,7 +13,6 @@ import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dsm.DsmService; @@ -22,7 +20,7 @@ import eu.dnetlib.errors.DsmException; import eu.dnetlib.errors.WorkflowManagerException; import eu.dnetlib.is.model.resource.SimpleResource; import eu.dnetlib.is.resource.repository.SimpleResourceRepository; -import eu.dnetlib.manager.wf.model.GraphNode; +import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; @@ -111,38 +109,34 @@ public class WorkflowExecutor implements Stoppable { return startWorkflowInstance(instance, processCallback, parent); } - public String startWorkflowInstance(final WorkflowInstance instance, final ProcessCallback processCallback, final String parent) + public String startWorkflowInstance(final WorkflowInstance wfInstance, final ProcessCallback processCallback, final String parent) throws WorkflowManagerException { - if (!instance.isEnabled() || !instance.isConfigured()) { - log.warn("Wf instance " + instance.getId() + " is not ready to start"); - throw new WorkflowManagerException("Wf instance " + instance.getId() + " is not ready to start"); + if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) { + log.warn("Wf instance " + wfInstance.getId() + " is not ready to start"); + throw new WorkflowManagerException("Wf instance " + wfInstance.getId() + " is not ready to start"); } - final SimpleResource wf = simpleResourceRepository - .findById(instance.getWorkflow()) + final SimpleResource wfMetadata = simpleResourceRepository + .findById(wfInstance.getWorkflow()) .filter(r -> r.getType().equals("workflows")) - .orElseThrow(() -> new WorkflowManagerException("WF not found: " + instance.getWorkflow())); + .orElseThrow(() -> new WorkflowManagerException("WF not found: " + wfInstance.getWorkflow())); - final List graph = simpleResourceRepository.findContentById(wf.getId()) + final WorkflowGraph wfGraph = simpleResourceRepository.findContentById(wfMetadata.getId()) .map(s -> { - final ObjectMapper mapper = new ObjectMapper(); - final TypeReference> type = new TypeReference>() {}; - final List list = new ArrayList<>(); try { - list.addAll(mapper.readValue(s, type)); + return new ObjectMapper().readValue(s, WorkflowGraph.class); } catch (final Exception e) { - log.error("Error parsing json", e); + return (WorkflowGraph) null; } - return list; }) - .filter(list -> !list.isEmpty()) - .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wf.getId())); + .filter(Objects::nonNull) + .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId())); final WorkflowProcess process = - processFactory.newProcess(wf.getId(), wf.getName(), wf.getSubtype(), graph, instance, processCallback, parent); + processFactory.newProcess(wfMetadata, wfGraph, wfInstance, processCallback, parent); - return processRegistry.registerProcess(process, instance.getId()); + return processRegistry.registerProcess(process, wfInstance); } @Override diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index 9a479806..a62bc6bd 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -8,6 +8,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.lang3.math.NumberUtils; +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.Graph; import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; @@ -16,104 +18,96 @@ import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; */ public class WorkflowProcess implements Comparable { + public enum Status { + CREATED, + EXECUTING, + SUCCESS, + FAILURE, + KILLED; + } + + public enum StartMode { + AUTO, + MANUAL, + DISABLED + } + private final String id; - private final String name; - private final String family; - private String dsId; - private String dsName; - private String dsInterface; + private final SimpleResource wfMetadata; + private final WorkflowInstance wfInstance; private final Graph graph; private final ProcessCallback callback; - private final int priority; private final Env env; private final List tokens = new CopyOnWriteArrayList<>(); private LocalDateTime lastActivityDate; private Status status; private LocalDateTime startDate = LocalDateTime.MIN; private LocalDateTime endDate = LocalDateTime.MIN; - private final String wfId; - private final String wfInstanceId; - private Map> pausedJoinNodeTokens = new HashMap<>(); - private Map globalParams; + private final Map> pausedJoinNodeTokens = new HashMap<>(); + private final Map globalParams; private String error; private String errorStacktrace; - private Map outputParams = new HashMap<>(); - private String parentProfileId; + private final Map outputParams = new HashMap<>(); + private final String parentProfileId; public WorkflowProcess( final String id, - final String name, - final String family, - final String dsId, - final String dsName, - final String dsInterface, + final SimpleResource wfMetadata, + final WorkflowInstance wfInstance, final Graph graph, - final int priority, - final String wfId, - final String wfInstanceId, final Map globalParams, final ProcessCallback callback, final String parentProfileId) { this.id = id; - this.name = name; - this.family = family; - this.dsId = dsId; - this.dsName = dsName; - this.dsInterface = dsInterface; + this.wfMetadata = wfMetadata; + this.wfInstance = wfInstance; this.graph = graph; - this.priority = priority; this.callback = callback; this.status = Status.CREATED; this.env = new Env(); - this.wfId = wfId; - this.wfInstanceId = wfInstanceId; this.globalParams = globalParams; this.lastActivityDate = LocalDateTime.now(); this.parentProfileId = parentProfileId; } - public void setParentProfileId(final String parentProfileId) { - this.parentProfileId = parentProfileId; - } - - public String getDsId() { - return dsId; - } - - public void setDsId(final String dsId) { - this.dsId = dsId; - } - - public String getDsName() { - return dsName; - } - - public void setDsName(final String dsName) { - this.dsName = dsName; - } - - public String getDsInterface() { - return dsInterface; - } - - public void setDsInterface(final String dsInterface) { - this.dsInterface = dsInterface; - } - - public Map> getPausedJoinNodeTokens() { - return pausedJoinNodeTokens; - } - - public void setPausedJoinNodeTokens(final Map> pausedJoinNodeTokens) { - this.pausedJoinNodeTokens = pausedJoinNodeTokens; - } - public String getId() { return id; } public String getName() { - return name; + return wfMetadata.getName(); + } + + public String getFamily() { + return wfMetadata.getSubtype(); + } + + public String getWfId() { + return wfMetadata.getId(); + } + + public String getWfInstanceId() { + return wfInstance.getId(); + } + + public int getPriority() { + return wfInstance.getPriority(); + } + + public String getDsId() { + return wfInstance.getId(); + } + + public String getDsName() { + return wfInstance.getDsName(); + } + + public String getDsInterface() { + return wfInstance.getApiId(); + } + + public Map> getPausedJoinNodeTokens() { + return pausedJoinNodeTokens; } public Env getEnv() { @@ -144,10 +138,6 @@ public class WorkflowProcess implements Comparable { return callback; } - public int getPriority() { - return priority; - } - public boolean isTerminated() { switch (status) { case SUCCESS: @@ -169,7 +159,7 @@ public class WorkflowProcess implements Comparable { @Override public String toString() { - return String.format("[process id='%s' name='%s']", id, name); + return String.format("[process id='%s' name='%s']", id, wfMetadata.getName()); } @Override @@ -181,22 +171,6 @@ public class WorkflowProcess implements Comparable { return globalParams; } - public void setGlobalParams(final Map globalParams) { - this.globalParams = globalParams; - } - - public String getFamily() { - return family; - } - - public String getWfId() { - return wfId; - } - - public String getWfInstanceId() { - return wfInstanceId; - } - public void setStartDate(final LocalDateTime startDate) { this.startDate = startDate; } @@ -213,20 +187,6 @@ public class WorkflowProcess implements Comparable { return endDate; } - public enum Status { - CREATED, - EXECUTING, - SUCCESS, - FAILURE, - KILLED; - } - - public enum StartMode { - AUTO, - MANUAL, - DISABLED - } - public String getError() { return error; } @@ -243,10 +203,6 @@ public class WorkflowProcess implements Comparable { this.errorStacktrace = errorStacktrace; } - public void setOutputParams(final Map outputParams) { - this.outputParams = outputParams; - } - public Map getOutputParams() { return outputParams; } From 30991cabbfcaec45d63ea6b51d6e871a97b7658c Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 8 Mar 2023 08:08:26 +0100 Subject: [PATCH 11/19] refactoring --- .../manager/wf/model/WorkflowGraph.java | 17 ++++++------ .../manager/wf/nodes/AbstractJobNode.java | 7 ++--- .../manager/wf/nodes/DefaultJobNode.java | 2 -- .../wf/nodes/LaunchWorkflowJobNode.java | 2 -- .../manager/wf/workflows/graph/Arc.java | 27 +++++++++++-------- .../manager/wf/workflows/graph/Graph.java | 20 ++++++++------ .../wf/workflows/graph/GraphLoader.java | 11 ++++++-- .../wf/workflows/procs/ProcessEngine.java | 2 +- .../manager/wf/workflows/procs/Token.java | 19 ------------- 9 files changed, 48 insertions(+), 59 deletions(-) diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java index eae06aa1..76fcc054 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java @@ -194,16 +194,8 @@ public class WorkflowGraph implements Serializable { private static final long serialVersionUID = 7866138976929522262L; - private String name; private String to; - - public String getName() { - return name; - } - - public void setName(final String name) { - this.name = name; - } + private String condition; public String getTo() { return to; @@ -213,6 +205,13 @@ public class WorkflowGraph implements Serializable { this.to = to; } + public String getCondition() { + return condition; + } + + public void setCondition(final String condition) { + this.condition = condition; + } } class NodeParam implements Serializable { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java index 3e8807f3..e74cff41 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java @@ -14,12 +14,9 @@ public abstract class AbstractJobNode extends ProcessNode { try { log.debug("START NODE: " + getBeanName()); beforeStart(token); - final String arc = execute(token.getEnv()); + execute(token.getEnv()); beforeCompleted(token); log.debug("END NODE (SUCCESS): " + getBeanName()); - - token.release(arc); - } catch (final Throwable e) { log.error("got exception while executing workflow node", e); log.debug("END NODE (FAILED): " + getBeanName()); @@ -28,7 +25,7 @@ public abstract class AbstractJobNode extends ProcessNode { } } - abstract protected String execute(final Env env) throws Exception; + abstract protected void execute(final Env env) throws Exception; protected void beforeStart(final Token token) { // For optional overwrites diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java index 32a8c750..c3d97e53 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java @@ -1,6 +1,5 @@ package eu.dnetlib.manager.wf.nodes; -import eu.dnetlib.manager.wf.workflows.graph.Arc; import eu.dnetlib.manager.wf.workflows.procs.Token; /** @@ -15,7 +14,6 @@ public final class DefaultJobNode extends ProcessNode { @Override public void execute(final Token token) { - token.setNextArc(Arc.DEFAULT_ARC); token.release(); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index 1818bda1..8b9f758a 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -4,7 +4,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; -import eu.dnetlib.manager.wf.workflows.graph.Arc; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; import eu.dnetlib.manager.wf.workflows.procs.Token; @@ -39,7 +38,6 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { @Override public void onSuccess() { log.debug("Child workflow has been completed successfully"); - token.setNextArc(Arc.DEFAULT_ARC); token.release(); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java index a0347f36..1d60254b 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java @@ -1,23 +1,19 @@ package eu.dnetlib.manager.wf.workflows.graph; -import org.apache.commons.lang3.StringUtils; +import com.google.common.base.Function; + +import eu.dnetlib.manager.wf.workflows.procs.Env; public class Arc { - public static final String DEFAULT_ARC = null; - - private final String name; private final String from; private final String to; + private final Function condFunction; - public Arc(final String name, final String from, final String to) { - this.name = name; + public Arc(final String from, final String to, final Function condFunction) { this.from = from; this.to = to; - } - - public String getName() { - return this.name; + this.condFunction = condFunction; } public String getFrom() { @@ -28,8 +24,17 @@ public class Arc { return this.to; } + public boolean isValid(final Env env) { + if (condFunction != null) { + return condFunction.apply(env); + } else { + return true; + } + } + @Override public String toString() { - return String.format("[ %s: %s -> %s ]", StringUtils.isBlank(this.name) ? "DEFAULT" : this.name, this.from, this.to); + return String.format("[ %s -> %s, %s ]", this.from, this.to, this.condFunction != null ? "with cond" : "without cond"); } + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java index c9027e7f..a09b5f91 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java @@ -7,9 +7,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import eu.dnetlib.manager.wf.workflows.procs.Env; + public class Graph { private final Map nodes = new HashMap<>(); @@ -53,14 +56,15 @@ public class Graph { return res; } - public Set nextNodes(final GraphNode current, final String arcName) { - final Set res = new HashSet<>(); - for (final Arc arc : this.arcs) { - if (StringUtils.equals(arc.getFrom(), current.getName()) && StringUtils.equals(arc.getName(), arcName)) { - res.add(this.nodes.get(arc.getTo())); - } - } - return res; + public Set nextNodes(final GraphNode current, final Env env) { + return arcs.stream() + .filter(arc -> StringUtils.equals(arc.getFrom(), current.getName())) + .filter(arc -> arc.isValid(env)) + .map(arc -> arc.getTo()) + .distinct() + .map(to -> nodes.get(to)) + .collect(Collectors.toSet()); + } public int getNumberOfIncomingArcs(final GraphNode node) { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java index 6af1f94d..0b4c23e9 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -9,10 +9,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; +import com.google.common.base.Function; import com.google.common.collect.Sets; import eu.dnetlib.errors.WorkflowManagerException; import eu.dnetlib.manager.wf.model.WorkflowGraph; +import eu.dnetlib.manager.wf.workflows.procs.Env; import eu.dnetlib.manager.wf.workflows.util.NodeHelper; @Service @@ -47,9 +49,9 @@ public class GraphLoader { if (node.getArcs() != null) { for (final WorkflowGraph.Arc a : node.getArcs()) { - final String arcName = a.getName(); final String to = a.getTo(); - graph.addArc(new Arc(StringUtils.isNotBlank(arcName) ? arcName : Arc.DEFAULT_ARC, nodeName, to)); + final Function condFunction = generateFunction(a.getCondition()); + graph.addArc(new Arc(nodeName, to, condFunction)); } } @@ -61,6 +63,11 @@ public class GraphLoader { return graph; } + private Function generateFunction(final String condition) { + // TODO Auto-generated method stub + return null; + } + private void checkValidity(final Graph graph) throws WorkflowManagerException { final Set nodesFromArcs = new HashSet<>(); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java index 16c8bc50..50eead65 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -68,7 +68,7 @@ public class ProcessEngine { process.setLastActivityDate(LocalDateTime.now()); try { - for (final GraphNode node : process.getGraph().nextNodes(oldGraphNode, oldToken.getNextArc())) { + for (final GraphNode node : process.getGraph().nextNodes(oldGraphNode, oldToken.getEnv())) { if (node.isJoin() || node.isSucessNode()) { if (!process.getPausedJoinNodeTokens().containsKey(node.getName())) { process.getPausedJoinNodeTokens().put(node.getName(), new ArrayList()); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java index 9524dfe1..3b565dc1 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -5,7 +5,6 @@ import java.util.UUID; import com.google.common.base.Throwables; -import eu.dnetlib.manager.wf.workflows.graph.Arc; import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback; import eu.dnetlib.manager.wf.workflows.util.ProgressProvider; @@ -21,7 +20,6 @@ public class Token { private final NodeTokenCallback callback; private boolean failed = false; private LocalDateTime endDate = LocalDateTime.MIN; - private String nextArc = Arc.DEFAULT_ARC; private boolean active = true; private String error = ""; private String errorStackTrace = ""; @@ -54,14 +52,6 @@ public class Token { this.endDate = endDate; } - public String getNextArc() { - return this.nextArc; - } - - public void setNextArc(final String nextArc) { - this.nextArc = nextArc; - } - public boolean isActive() { return this.active; } @@ -78,15 +68,6 @@ public class Token { this.failed = failed; } - public void release(final String arcName) { - setNextArc(arcName); - setEndDate(LocalDateTime.now()); - setActive(false); - if (this.callback != null) { - this.callback.onSuccess(this); - } - } - public void release() { setEndDate(LocalDateTime.now()); setActive(false); From cd5e1ab957fc00d09bf1380d14a13375afae956f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 8 Mar 2023 09:39:37 +0100 Subject: [PATCH 12/19] refactoring --- .../manager/history/WorkflowLogger.java | 4 +- .../wf/cron/ScheduledWorkflowLauncher.java | 5 +- .../manager/wf/nodes/AbstractJobNode.java | 3 + .../wf/nodes/LaunchWorkflowJobNode.java | 29 +++--- .../dnetlib/manager/wf/nodes/ProcessNode.java | 10 ++ .../wf/workflows/procs/ProcessEngine.java | 98 +++---------------- .../wf/workflows/procs/ProcessFactory.java | 9 +- .../manager/wf/workflows/procs/Token.java | 52 +++++----- .../wf/workflows/procs/WorkflowExecutor.java | 25 +++-- .../wf/workflows/procs/WorkflowProcess.java | 72 ++++++++++++-- .../wf/workflows/util/NodeTokenCallback.java | 13 --- .../wf/workflows/util/ProcessCallback.java | 12 --- .../wf/workflows/util/ProgressProvider.java | 6 -- .../util/SubWorkflowProgressProvider.java | 37 ------- 14 files changed, 150 insertions(+), 225 deletions(-) delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java index c1dd8fd8..6212f9a7 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java @@ -40,8 +40,8 @@ public class WorkflowLogger { return wfProcessExecutionRepository.findById(processId).get(); } - public void saveProcessExecution(final WfProcessExecution proc) { - wfProcessExecutionRepository.save(proc); + public void saveProcessExecution(final WfProcessExecution pe) { + wfProcessExecutionRepository.save(pe); } public Optional getLastExecutionForInstance(final String id) { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 0eec4f58..3551d986 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -17,15 +17,12 @@ import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; -import eu.dnetlib.utils.DateUtils; @Service public class ScheduledWorkflowLauncher { private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class); - private static final DateUtils dateUtils = new DateUtils(); - @Autowired private WorkflowExecutor workflowExecutor; @@ -54,7 +51,7 @@ public class ScheduledWorkflowLauncher { .filter(this::isReady) .forEach(instance -> { try { - workflowExecutor.startWorkflowInstance(instance, null, null); + workflowExecutor.startWorkflowInstance(instance, null, null, null); } catch (final Exception e) { log.error("Error launching scheduled wf instance: " + instance.getId(), e); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java index e74cff41..212fd659 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java @@ -13,9 +13,12 @@ public abstract class AbstractJobNode extends ProcessNode { protected void doExecute(final Token token) { try { log.debug("START NODE: " + getBeanName()); + setProgressMessage(getNodeName()); + beforeStart(token); execute(token.getEnv()); beforeCompleted(token); + log.debug("END NODE (SUCCESS): " + getBeanName()); } catch (final Throwable e) { log.error("got exception while executing workflow node", e); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index 8b9f758a..f642d589 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -9,8 +9,6 @@ import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; import eu.dnetlib.manager.wf.workflows.procs.Token; import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; -import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; -import eu.dnetlib.manager.wf.workflows.util.SubWorkflowProgressProvider; /** * Created by michele on 18/11/15. @@ -33,28 +31,23 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { public final void execute(final Token token) { try { - final String procId = executor.startWorkflowInstance(getWfInstanceId(), new ProcessCallback() { - - @Override - public void onSuccess() { - log.debug("Child workflow has been completed successfully"); - token.release(); - } - - @Override - public void onFail() { - log.error("Child workflow is failed"); - token.releaseAsFailed("Child workflow is failed"); - } - }, process.getWfInstanceId()); + final String procId = executor.startWorkflowInstance(getWfInstanceId(), process.getWfInstanceId(), (proc) -> { + log.debug("Child workflow has been completed successfully"); + token.release(); + }, (proc) -> { + log.error("Child workflow is failed"); + token.releaseAsFailed("Child workflow is failed"); + }); if (log.isDebugEnabled()) { log.debug("The child workflow [instance: " + getWfInstanceId() + "] is starting with procId: " + procId); } - token.setProgressProvider(new SubWorkflowProgressProvider(procId, processRegistry)); + setProgressMessage("Launched sub workflow, proc: " + procId); - } catch (final Throwable e) { + } catch ( + + final Throwable e) { log.error("got exception while launching child workflow", e); token.releaseAsFailed(e); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java index 5052bb31..48664307 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java @@ -13,6 +13,8 @@ public abstract class ProcessNode implements BeanNameAware { private String nodeName; + private String progressMessage; + public abstract void execute(final Token token); public String getBeanName() { @@ -36,4 +38,12 @@ public abstract class ProcessNode implements BeanNameAware { public String toString() { return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName); } + + public String getProgressMessage() { + return progressMessage; + } + + public void setProgressMessage(final String progressMessage) { + this.progressMessage = progressMessage; + } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java index 50eead65..3755628a 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -4,11 +4,9 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -17,14 +15,10 @@ import org.springframework.stereotype.Service; import com.google.common.base.Throwables; import eu.dnetlib.manager.history.WorkflowLogger; -import eu.dnetlib.manager.history.model.WfProcessExecution; import eu.dnetlib.manager.wf.nodes.ProcessNode; import eu.dnetlib.manager.wf.notification.EmailSender; import eu.dnetlib.manager.wf.workflows.graph.GraphNode; -import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status; import eu.dnetlib.manager.wf.workflows.util.NodeHelper; -import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback; -import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; @Service public class ProcessEngine { @@ -51,8 +45,7 @@ public class ProcessEngine { try { for (final GraphNode node : process.getGraph().startNodes()) { final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv()); - final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); - + final Token token = prepareNewToken(process, node); token.getEnv().addAttributes(process.getEnv().getAttributes()); process.getTokens().add(token); @@ -79,7 +72,7 @@ public class ProcessEngine { list.add(oldToken); if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) { - final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); + final Token token = prepareNewToken(process, node); token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()]))); final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); @@ -87,13 +80,13 @@ public class ProcessEngine { process.setLastActivityDate(LocalDateTime.now()); if (node.isSucessNode()) { - markAsCompleted(process, token); + completeProcess(process, token); } else { pNode.execute(token); } } } else { - final Token token = new Token(node.getName(), newNodeTokenCallback(process, node)); + final Token token = prepareNewToken(process, node); token.getEnv().addAttributes(oldToken.getEnv().getAttributes()); final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); @@ -112,19 +105,10 @@ public class ProcessEngine { } - private NodeTokenCallback newNodeTokenCallback(final WorkflowProcess process, final GraphNode node) { - return new NodeTokenCallback() { - - @Override - public void onSuccess(final Token token) { - releaseToken(process, node, token); - } - - @Override - public void onFail(final Token token) { - completeProcess(process, token); - } - }; + private Token prepareNewToken(final WorkflowProcess process, final GraphNode node) { + return new Token( + token -> releaseToken(process, node, token), + token -> completeProcess(process, token)); } private Map mergeEnvParams(final Token... tokens) { @@ -133,71 +117,11 @@ public class ProcessEngine { return map; } - private void markAsCompleted(final WorkflowProcess process, final Token token) { - completeProcess(process, token); - } - private void completeProcess(final WorkflowProcess process, final Token token) { - if (token.isActive()) { - if (StringUtils.isNotBlank(token.getError())) { - token.releaseAsFailed(token.getError()); - } else { - token.release(); - } - } - - final LocalDateTime now = token.getEndDate(); - - process.setLastActivityDate(now); - process.setEndDate(now); - process.setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS); - - if (token.isFailed()) { - process.setStatus(Status.FAILURE); - process.setError(token.getError()); - process.setErrorStacktrace(token.getErrorStackTrace()); - process.setLastActivityDate(LocalDateTime.now()); - } - - if (process.getCallback() != null) { - if (token.isFailed()) { - process.getCallback().onFail(); - } else { - process.getCallback().onSuccess(); - } - } - - final Map details = new LinkedHashMap<>(); - details.putAll(process.getOutputParams()); - details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + process.getPriority()); - details.put(WorkflowsConstants.LOG_WF_ID, process.getWfId()); - details.put(WorkflowsConstants.LOG_WF_ID, process.getWfInstanceId()); - - if (process.getError() != null) { - details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, process.getError()); - details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, process.getErrorStacktrace()); - } - - final WfProcessExecution pe = new WfProcessExecution(); - pe.setProcessId(process.getId()); - pe.setName(process.getName()); - pe.setFamily(process.getFamily()); - - pe.setDsId(process.getDsId()); - pe.setDsName(process.getDsName()); - pe.setDsApi(process.getDsInterface()); - - pe.setStartDate(process.getStartDate()); - pe.setEndDate(process.getEndDate()); - - pe.setStatus(process.getStatus().toString()); - - pe.setDetails(details); - - wfLogger.saveProcessExecution(pe); - + token.checkStatus(); + process.complete(token); + wfLogger.saveProcessExecution(process.asLog()); emailSender.sendMails(process); - } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index c7c07ace..c6479f17 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -4,6 +4,7 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; +import java.util.function.Consumer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -16,7 +17,6 @@ import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.Graph; import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; -import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; @Component public class ProcessFactory { @@ -33,8 +33,9 @@ public class ProcessFactory { public WorkflowProcess newProcess(final SimpleResource wfMetadata, final WorkflowGraph wfGraph, final WorkflowInstance wfInstance, - final ProcessCallback processCallback, - final String parent) throws WorkflowManagerException { + final String parent, + final Consumer onSuccess, + final Consumer onFail) throws WorkflowManagerException { final Map globalParams = new HashMap<>(); globalParams.putAll(wfInstance.getSystemParams()); @@ -42,7 +43,7 @@ public class ProcessFactory { final Graph graph = graphLoader.loadGraph(wfGraph, globalParams); - return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, processCallback, parent); + return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, parent, onSuccess, onFail); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java index 3b565dc1..c4635939 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -2,34 +2,36 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; import java.util.UUID; +import java.util.function.Consumer; + +import org.apache.commons.lang3.StringUtils; import com.google.common.base.Throwables; -import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback; -import eu.dnetlib.manager.wf.workflows.util.ProgressProvider; - /** * Created by michele on 19/11/15. */ public class Token { private final String id; - private final String nodeName; private final Env env = new Env(); private final LocalDateTime startDate; - private final NodeTokenCallback callback; - private boolean failed = false; private LocalDateTime endDate = LocalDateTime.MIN; + + private final Consumer onSuccess; + private final Consumer onFail; + + private boolean failed = false; + private boolean active = true; private String error = ""; private String errorStackTrace = ""; - private ProgressProvider progressProvider; - public Token(final String nodeName, final NodeTokenCallback callback) { + public Token(final Consumer onSuccess, final Consumer onFail) { this.id = "token-" + UUID.randomUUID(); - this.nodeName = nodeName; this.startDate = LocalDateTime.now(); - this.callback = callback; + this.onSuccess = onSuccess; + this.onFail = onFail; } public String getId() { @@ -71,8 +73,8 @@ public class Token { public void release() { setEndDate(LocalDateTime.now()); setActive(false); - if (this.callback != null) { - this.callback.onSuccess(this); + if (this.onSuccess != null) { + this.onSuccess.accept(this); } } @@ -82,8 +84,8 @@ public class Token { setFailed(true); setError(e.getMessage()); setErrorStackTrace(Throwables.getStackTraceAsString(e)); - if (this.callback != null) { - this.callback.onFail(this); + if (this.onFail != null) { + this.onFail.accept(this); } } @@ -92,13 +94,19 @@ public class Token { setActive(false); setFailed(true); setError(error); - if (this.callback != null) { - this.callback.onFail(this); + if (this.onFail != null) { + this.onFail.accept(this); } } - public String getNodeName() { - return this.nodeName; + public void checkStatus() { + if (isActive()) { + if (StringUtils.isNotBlank(error)) { + releaseAsFailed(error); + } else { + release(); + } + } } public String getError() { @@ -117,12 +125,4 @@ public class Token { this.errorStackTrace = errorStackTrace; } - public ProgressProvider getProgressProvider() { - return this.progressProvider; - } - - public void setProgressProvider(final ProgressProvider progressProvider) { - this.progressProvider = progressProvider; - } - } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java index e83add35..dc1a727c 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -5,6 +5,7 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import javax.annotation.PostConstruct; @@ -23,7 +24,6 @@ import eu.dnetlib.is.resource.repository.SimpleResourceRepository; import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; -import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; import eu.dnetlib.utils.Stoppable; import eu.dnetlib.utils.StoppableDetails; @@ -63,7 +63,12 @@ public class WorkflowExecutor implements Stoppable { }, 10, 10, TimeUnit.SECONDS); } - public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, final ProcessCallback processCallback, final String parent) + public String startRepoHiWorkflow(final String wfId, + final String dsId, + final String apiId, + final String parent, + final Consumer onSuccess, + final Consumer onFail) throws WorkflowManagerException { if (isPaused()) { @@ -91,13 +96,16 @@ public class WorkflowExecutor implements Stoppable { instance.setSystemParams(new HashMap<>()); instance.setUserParams(new HashMap<>()); - return startWorkflowInstance(instance, processCallback, parent); + return startWorkflowInstance(instance, parent, onSuccess, onFail); } catch (final DsmException e) { throw new WorkflowManagerException("Invalid datasource: " + dsId, e); } } - public String startWorkflowInstance(final String wfInstanceId, final ProcessCallback processCallback, final String parent) throws Exception { + public String startWorkflowInstance(final String wfInstanceId, + final String parent, + final Consumer onSuccess, + final Consumer onFail) throws Exception { if (isPaused()) { log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown"); @@ -106,10 +114,13 @@ public class WorkflowExecutor implements Stoppable { final WorkflowInstance instance = workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId)); - return startWorkflowInstance(instance, processCallback, parent); + return startWorkflowInstance(instance, parent, onSuccess, onFail); } - public String startWorkflowInstance(final WorkflowInstance wfInstance, final ProcessCallback processCallback, final String parent) + public String startWorkflowInstance(final WorkflowInstance wfInstance, + final String parent, + final Consumer onSuccess, + final Consumer onFail) throws WorkflowManagerException { if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) { @@ -134,7 +145,7 @@ public class WorkflowExecutor implements Stoppable { .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId())); final WorkflowProcess process = - processFactory.newProcess(wfMetadata, wfGraph, wfInstance, processCallback, parent); + processFactory.newProcess(wfMetadata, wfGraph, wfInstance, parent, onSuccess, onFail); return processRegistry.registerProcess(process, wfInstance); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index a62bc6bd..2b7f09a8 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -2,16 +2,19 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; import org.apache.commons.lang3.math.NumberUtils; import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.manager.history.model.WfProcessExecution; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.Graph; -import eu.dnetlib.manager.wf.workflows.util.ProcessCallback; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; /** * Created by michele on 19/11/15. @@ -36,7 +39,8 @@ public class WorkflowProcess implements Comparable { private final SimpleResource wfMetadata; private final WorkflowInstance wfInstance; private final Graph graph; - private final ProcessCallback callback; + private final Consumer onSuccess; + private final Consumer onFail; private final Env env; private final List tokens = new CopyOnWriteArrayList<>(); private LocalDateTime lastActivityDate; @@ -56,13 +60,15 @@ public class WorkflowProcess implements Comparable { final WorkflowInstance wfInstance, final Graph graph, final Map globalParams, - final ProcessCallback callback, - final String parentProfileId) { + final String parentProfileId, + final Consumer onSuccess, + final Consumer onFail) { this.id = id; this.wfMetadata = wfMetadata; this.wfInstance = wfInstance; this.graph = graph; - this.callback = callback; + this.onSuccess = onSuccess; + this.onFail = onFail; this.status = Status.CREATED; this.env = new Env(); this.globalParams = globalParams; @@ -134,10 +140,6 @@ public class WorkflowProcess implements Comparable { setStatus(Status.KILLED); } - public ProcessCallback getCallback() { - return callback; - } - public boolean isTerminated() { switch (status) { case SUCCESS: @@ -211,4 +213,56 @@ public class WorkflowProcess implements Comparable { return parentProfileId; } + public void complete(final Token token) { + final LocalDateTime now = token.getEndDate(); + setLastActivityDate(now); + setEndDate(now); + setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS); + + if (token.isFailed()) { + setStatus(Status.FAILURE); + setError(token.getError()); + setErrorStacktrace(token.getErrorStackTrace()); + setLastActivityDate(LocalDateTime.now()); + } + + if (token.isFailed() && onFail != null) { + onFail.accept(this); + } + if (!token.isFailed() && onSuccess != null) { + onSuccess.accept(this);; + } + + } + + public WfProcessExecution asLog() { + final Map details = new LinkedHashMap<>(); + details.putAll(getOutputParams()); + details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + getPriority()); + details.put(WorkflowsConstants.LOG_WF_ID, getWfId()); + details.put(WorkflowsConstants.LOG_WF_ID, getWfInstanceId()); + + if (getError() != null) { + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, getError()); + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, getErrorStacktrace()); + } + + final WfProcessExecution pe = new WfProcessExecution(); + pe.setProcessId(getId()); + pe.setName(getName()); + pe.setFamily(getFamily()); + + pe.setDsId(getDsId()); + pe.setDsName(getDsName()); + pe.setDsApi(getDsInterface()); + + pe.setStartDate(getStartDate()); + pe.setEndDate(getEndDate()); + pe.setStatus(getStatus().toString()); + + pe.setDetails(details); + + return pe; + } + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java deleted file mode 100644 index 7496186b..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeTokenCallback.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.dnetlib.manager.wf.workflows.util; - -import eu.dnetlib.manager.wf.workflows.procs.Token; - -/** - * Created by michele on 26/11/15. - */ -public interface NodeTokenCallback { - - void onSuccess(Token token); - - void onFail(Token token); -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java deleted file mode 100644 index 4f70b5da..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProcessCallback.java +++ /dev/null @@ -1,12 +0,0 @@ -package eu.dnetlib.manager.wf.workflows.util; - -/** - * Created by michele on 18/11/15. - */ -public interface ProcessCallback { - - void onSuccess(); - - void onFail(); - -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java deleted file mode 100644 index d9f84f45..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ProgressProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package eu.dnetlib.manager.wf.workflows.util; - -public interface ProgressProvider { - - String getProgressDescription(); -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java deleted file mode 100644 index 499349a3..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/SubWorkflowProgressProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -package eu.dnetlib.manager.wf.workflows.util; - -import java.util.List; -import java.util.stream.Collectors; - -import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; -import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; - -public class SubWorkflowProgressProvider implements ProgressProvider { - - private final String procId; - private final ProcessRegistry processRegistry; - - public SubWorkflowProgressProvider(final String procId, final ProcessRegistry processRegistry) { - super(); - this.procId = procId; - this.processRegistry = processRegistry; - } - - @Override - public String getProgressDescription() { - final WorkflowProcess proc = this.processRegistry.findProcess(this.procId); - - if (proc == null) { return "-"; } - - final List list = proc.getTokens() - .stream() - .filter(t -> t.isActive()) - .map(t -> t.getProgressProvider() != null ? String.format("%s (%s)", t.getNodeName(), t.getProgressProvider().getProgressDescription()) - : t.getNodeName()) - .collect(Collectors.toList()); - if (!list.isEmpty()) { return list.stream().collect(Collectors.joining(", ")); } - - return "-"; - } - -} From 0782c2f536b40d63e569b4f4ac935173185ab2e3 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 8 Mar 2023 10:10:03 +0100 Subject: [PATCH 13/19] refactoring --- .../manager/wf/model/WorkflowInstance.java | 12 +++++ .../wf/cron/ScheduledWorkflowLauncher.java | 2 +- .../wf/nodes/LaunchWorkflowJobNode.java | 52 ++++++++++++------- .../wf/workflows/procs/ProcessFactory.java | 3 +- .../wf/workflows/procs/WorkflowExecutor.java | 8 ++- .../wf/workflows/procs/WorkflowProcess.java | 11 ++-- 6 files changed, 55 insertions(+), 33 deletions(-) diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java index 53d5afd1..dca1ef1c 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -8,6 +8,7 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import javax.persistence.Transient; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; @@ -75,6 +76,9 @@ public class WorkflowInstance implements Serializable { @Column(name = "user_params", columnDefinition = "jsonb") private Map userParams = new LinkedHashMap<>(); + @Transient + private String parentId; + public String getId() { return id; } @@ -194,4 +198,12 @@ public class WorkflowInstance implements Serializable { public void setUserParams(final Map userParams) { this.userParams = userParams; } + + public String getParentId() { + return parentId; + } + + public void setParentId(final String parentId) { + this.parentId = parentId; + } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 3551d986..28f927f0 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -51,7 +51,7 @@ public class ScheduledWorkflowLauncher { .filter(this::isReady) .forEach(instance -> { try { - workflowExecutor.startWorkflowInstance(instance, null, null, null); + workflowExecutor.startWorkflowInstance(instance, null, null); } catch (final Exception e) { log.error("Error launching scheduled wf instance: " + instance.getId(), e); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index f642d589..663da422 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -1,11 +1,14 @@ package eu.dnetlib.manager.wf.nodes; +import java.util.HashMap; +import java.util.UUID; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; +import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; -import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; import eu.dnetlib.manager.wf.workflows.procs.Token; import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; @@ -17,21 +20,36 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { private static final Log log = LogFactory.getLog(LaunchWorkflowJobNode.class); - private String wfInstanceId; + private String wfId; @Autowired private WorkflowExecutor executor; - @Autowired - private ProcessRegistry processRegistry; - private WorkflowProcess process; @Override public final void execute(final Token token) { try { - final String procId = executor.startWorkflowInstance(getWfInstanceId(), process.getWfInstanceId(), (proc) -> { + final WorkflowInstance instance = new WorkflowInstance(); + instance.setId("CHILD_" + UUID.randomUUID()); + instance.setParentId(process.getWfInstanceId()); + instance.setDetails(new HashMap<>()); + instance.setPriority(100); + instance.setDsId(process.getDsId()); + instance.setDsName(process.getDsName()); + instance.setApiId(process.getDsInterface()); + instance.setEnabled(true); + instance.setConfigured(true); + instance.setSchedulingEnabled(false); + instance.setCronExpression(""); + instance.setCronMinInterval(0); + instance.setWorkflow(wfId); + instance.setDestroyWf(null); + instance.setSystemParams(process.getGlobalParams()); + instance.setUserParams(new HashMap<>()); + + final String procId = executor.startWorkflowInstance(instance, (proc) -> { log.debug("Child workflow has been completed successfully"); token.release(); }, (proc) -> { @@ -40,30 +58,28 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { }); if (log.isDebugEnabled()) { - log.debug("The child workflow [instance: " + getWfInstanceId() + "] is starting with procId: " + procId); + log.debug("The child workflow [instance: " + instance.getId() + "] is starting with procId: " + procId); } setProgressMessage("Launched sub workflow, proc: " + procId); - } catch ( - - final Throwable e) { + } catch (final Throwable e) { log.error("got exception while launching child workflow", e); token.releaseAsFailed(e); } } - public String getWfInstanceId() { - return wfInstanceId; - } - - public void setWfInstanceId(final String wfInstanceId) { - this.wfInstanceId = wfInstanceId; - } - @Override public void setProcess(final WorkflowProcess process) { this.process = process; } + public String getWfId() { + return wfId; + } + + public void setWfId(final String wfId) { + this.wfId = wfId; + } + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index c6479f17..c700b1e2 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -33,7 +33,6 @@ public class ProcessFactory { public WorkflowProcess newProcess(final SimpleResource wfMetadata, final WorkflowGraph wfGraph, final WorkflowInstance wfInstance, - final String parent, final Consumer onSuccess, final Consumer onFail) throws WorkflowManagerException { @@ -43,7 +42,7 @@ public class ProcessFactory { final Graph graph = graphLoader.loadGraph(wfGraph, globalParams); - return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, parent, onSuccess, onFail); + return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, onSuccess, onFail); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java index dc1a727c..996abe97 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java @@ -66,7 +66,6 @@ public class WorkflowExecutor implements Stoppable { public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, - final String parent, final Consumer onSuccess, final Consumer onFail) throws WorkflowManagerException { @@ -96,7 +95,7 @@ public class WorkflowExecutor implements Stoppable { instance.setSystemParams(new HashMap<>()); instance.setUserParams(new HashMap<>()); - return startWorkflowInstance(instance, parent, onSuccess, onFail); + return startWorkflowInstance(instance, onSuccess, onFail); } catch (final DsmException e) { throw new WorkflowManagerException("Invalid datasource: " + dsId, e); } @@ -114,11 +113,10 @@ public class WorkflowExecutor implements Stoppable { final WorkflowInstance instance = workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId)); - return startWorkflowInstance(instance, parent, onSuccess, onFail); + return startWorkflowInstance(instance, onSuccess, onFail); } public String startWorkflowInstance(final WorkflowInstance wfInstance, - final String parent, final Consumer onSuccess, final Consumer onFail) throws WorkflowManagerException { @@ -145,7 +143,7 @@ public class WorkflowExecutor implements Stoppable { .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId())); final WorkflowProcess process = - processFactory.newProcess(wfMetadata, wfGraph, wfInstance, parent, onSuccess, onFail); + processFactory.newProcess(wfMetadata, wfGraph, wfInstance, onSuccess, onFail); return processRegistry.registerProcess(process, wfInstance); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index 2b7f09a8..78807371 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -52,7 +52,6 @@ public class WorkflowProcess implements Comparable { private String error; private String errorStacktrace; private final Map outputParams = new HashMap<>(); - private final String parentProfileId; public WorkflowProcess( final String id, @@ -60,7 +59,6 @@ public class WorkflowProcess implements Comparable { final WorkflowInstance wfInstance, final Graph graph, final Map globalParams, - final String parentProfileId, final Consumer onSuccess, final Consumer onFail) { this.id = id; @@ -73,7 +71,6 @@ public class WorkflowProcess implements Comparable { this.env = new Env(); this.globalParams = globalParams; this.lastActivityDate = LocalDateTime.now(); - this.parentProfileId = parentProfileId; } public String getId() { @@ -96,6 +93,10 @@ public class WorkflowProcess implements Comparable { return wfInstance.getId(); } + public String getParentId() { + return wfInstance.getParentId(); + } + public int getPriority() { return wfInstance.getPriority(); } @@ -209,10 +210,6 @@ public class WorkflowProcess implements Comparable { return outputParams; } - public String getParentProfileId() { - return parentProfileId; - } - public void complete(final Token token) { final LocalDateTime now = token.getEndDate(); setLastActivityDate(now); From 34f6e2977b94b4090c9c50601553594cd9f01041 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 10 Mar 2023 12:15:54 +0100 Subject: [PATCH 14/19] frontend: wf instances model --- .../src/app/app-routing.module.ts | 4 ++-- .../dnet-is-application/src/app/app.module.ts | 4 ++-- .../src/app/common/is.model.ts | 19 +++++++++++++++++++ .../wf-instance.dialog.html} | 0 .../wf-instances/wf-instances.component.css | 0 .../wf-instances.component.html} | 0 .../wf-instances/wf-instances.component.ts | 10 ++++++++++ .../src/app/wfs/wfs.component.ts | 10 ---------- 8 files changed, 33 insertions(+), 14 deletions(-) rename frontends/dnet-is-application/src/app/{wfs/wfs.component.css => wf-instances/wf-instance.dialog.html} (100%) create mode 100644 frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.css rename frontends/dnet-is-application/src/app/{wfs/wfs.component.html => wf-instances/wf-instances.component.html} (100%) create mode 100644 frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts delete mode 100644 frontends/dnet-is-application/src/app/wfs/wfs.component.ts diff --git a/frontends/dnet-is-application/src/app/app-routing.module.ts b/frontends/dnet-is-application/src/app/app-routing.module.ts index 3401870c..b2a848b5 100644 --- a/frontends/dnet-is-application/src/app/app-routing.module.ts +++ b/frontends/dnet-is-application/src/app/app-routing.module.ts @@ -10,7 +10,7 @@ import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/ import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; import { EmailsComponent } from './emails/emails.component'; -import { WfsComponent } from './wfs/wfs.component'; +import { WfInstancesComponent } from './wf-instances/wf-instances.component'; const routes: Routes = [ { path: "", redirectTo: 'info', pathMatch: 'full' }, @@ -20,7 +20,7 @@ const routes: Routes = [ { path: "adv_resources/vocabulary", component: VocabulariesComponent }, { path: "adv_resources/protocol", component: ProtocolsComponent }, { path: "adv_resources/email", component: EmailsComponent }, - { path: "wfs", component: WfsComponent }, + { path: "wfs", component: WfInstancesComponent }, { path: "wf_history", component: WfHistoryComponent }, { path: "ctx_viewer", component: ContextViewerComponent }, { path: "voc_editor", component: VocabularyEditorComponent }, diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 85955647..374a19b0 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -37,7 +37,7 @@ import { SpinnerHttpInterceptor } from './common/spinner.service'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; import { EmailDialog, EmailsComponent } from './emails/emails.component'; -import { WfsComponent } from './wfs/wfs.component'; +import { WfInstancesComponent } from './wf-instances/wf-instances.component'; @NgModule({ declarations: [ @@ -71,7 +71,7 @@ import { WfsComponent } from './wfs/wfs.component'; CleanerTesterComponent, EmailsComponent, EmailDialog, - WfsComponent + WfInstancesComponent ], imports: [ BrowserModule, diff --git a/frontends/dnet-is-application/src/app/common/is.model.ts b/frontends/dnet-is-application/src/app/common/is.model.ts index 7ea47b6f..59faf1e4 100644 --- a/frontends/dnet-is-application/src/app/common/is.model.ts +++ b/frontends/dnet-is-application/src/app/common/is.model.ts @@ -206,3 +206,22 @@ export interface EmailTemplate { subject: string, message: string } + + +export interface WfInstance { + id: string, + details: Map, + priority: number, + dsId?: string, + dsName?: string, + apiId?: string, + enabled: boolean, + configured: boolean, + schedulingEnabled: boolean, + cronExpression?: string, + cronMinInterval?: number, + workflow: string, + destroyWf?: string, + systemParams: Map, + userParams: Map +} diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.css b/frontends/dnet-is-application/src/app/wf-instances/wf-instance.dialog.html similarity index 100% rename from frontends/dnet-is-application/src/app/wfs/wfs.component.css rename to frontends/dnet-is-application/src/app/wf-instances/wf-instance.dialog.html diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.css b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.html b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html similarity index 100% rename from frontends/dnet-is-application/src/app/wfs/wfs.component.html rename to frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts new file mode 100644 index 00000000..da515868 --- /dev/null +++ b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts @@ -0,0 +1,10 @@ +import { Component } from '@angular/core'; + +@Component({ + selector: 'app-wf-instances', + templateUrl: './wf-instances.component.html', + styleUrls: ['./wf-instances.component.css'] +}) +export class WfInstancesComponent { + +} diff --git a/frontends/dnet-is-application/src/app/wfs/wfs.component.ts b/frontends/dnet-is-application/src/app/wfs/wfs.component.ts deleted file mode 100644 index 7b12ca91..00000000 --- a/frontends/dnet-is-application/src/app/wfs/wfs.component.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Component } from '@angular/core'; - -@Component({ - selector: 'app-wfs', - templateUrl: './wfs.component.html', - styleUrls: ['./wfs.component.css'] -}) -export class WfsComponent { - -} From 630b4c282d8f1a0a13b1966f5a3802e1970adc0a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 10 Mar 2023 15:20:49 +0100 Subject: [PATCH 15/19] minors --- apps/dnet-is-application/pom.xml | 6 ++++ .../manager/wf/WfHistoryAjaxController.java | 5 ++-- .../manager/wf/WfInstancesController.java | 30 +++++++++++++++++++ .../src/app/common/is.service.ts | 2 +- .../manager/wf/model/WorkflowInstance.java | 3 +- .../java/eu/dnetlib/utils/CountedValue.java | 13 ++++++++ .../WorkflowInstanceRepository.java | 10 +++++++ ...cutor.java => WorkflowManagerService.java} | 25 ++++++++++++---- .../wf/cron/ScheduledWorkflowLauncher.java | 10 +++---- .../wf/nodes/LaunchWorkflowJobNode.java | 6 ++-- .../wf/workflows/procs/ProcessRegistry.java | 2 +- .../manager/wf/workflows/util/NodeHelper.java | 5 ++-- 12 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java create mode 100644 libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java rename libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/{workflows/procs/WorkflowExecutor.java => WorkflowManagerService.java} (86%) diff --git a/apps/dnet-is-application/pom.xml b/apps/dnet-is-application/pom.xml index 27cb2137..92bde558 100644 --- a/apps/dnet-is-application/pom.xml +++ b/apps/dnet-is-application/pom.xml @@ -20,6 +20,12 @@ ${project.version} + + eu.dnetlib.dhp + dnet-wf-service + ${project.version} + + eu.dnetlib.dhp dnet-data-services diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java index dc4b18f5..0993426b 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java @@ -9,12 +9,13 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.manager.history.WorkflowLogger; import eu.dnetlib.manager.history.model.WfProcessExecution; @RestController -@RequestMapping("/ajax/wfs") -public class WfHistoryAjaxController { +@RequestMapping("/ajax/wf_history") +public class WfHistoryAjaxController extends AbstractDnetController { @Autowired private WorkflowLogger logger; diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java new file mode 100644 index 00000000..d8fc4096 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java @@ -0,0 +1,30 @@ +package eu.dnetlib.manager.wf; + +import java.util.List; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.utils.CountedValue; + +@RestController +@RequestMapping("/ajax/wf_instances") +public class WfInstancesController extends AbstractDnetController { + + private WorkflowManagerService wfManagerService; + + @GetMapping("/instance/{id}") + public WorkflowInstance getWfInstance(@PathVariable final String id) throws Exception { + return wfManagerService.findWorkflowInstance(id); + } + + @GetMapping("/families") + public List listWfFamilies() throws Exception { + return wfManagerService.families(); + } + +} diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index ed6297dc..339da869 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -99,7 +99,7 @@ export class ISService { if (from && from > 0) { params = params.append('from', from); } if (to && to > 0) { params = params.append('to', to); } - this.client.get('/ajax/wfs/', { params: params }).subscribe({ + this.client.get('/ajax/wf_history/', { params: params }).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java index dca1ef1c..0b6ff317 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -32,7 +32,7 @@ public class WorkflowInstance implements Serializable { private String id; @Type(type = "jsonb") - @Column(name = "data", columnDefinition = "jsonb") + @Column(name = "details", columnDefinition = "jsonb") private Map details = new LinkedHashMap<>(); @Column(name = "priority") @@ -206,4 +206,5 @@ public class WorkflowInstance implements Serializable { public void setParentId(final String parentId) { this.parentId = parentId; } + } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java new file mode 100644 index 00000000..046e6c6d --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java @@ -0,0 +1,13 @@ +package eu.dnetlib.utils; + +public interface CountedValue { + + public String getValue(); + + public void setValue(); + + public long getCount(); + + public void setCount(final long count); + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java index cb477905..e30934d5 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java @@ -1,9 +1,19 @@ package eu.dnetlib.manager.wf.repository; +import java.util.List; + import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.utils.CountedValue; public interface WorkflowInstanceRepository extends JpaRepository { + @Query(value = "select r.subtype as value, count(*) as count " + + "from workflow_instances i join resources r on (i.workflow = r.id) " + + "group by r.subtype " + + "order by count desc;", nativeQuery = true) + List families(); + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java similarity index 86% rename from libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java rename to libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java index 996abe97..82d7964c 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowExecutor.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -1,6 +1,7 @@ -package eu.dnetlib.manager.wf.workflows.procs; +package eu.dnetlib.manager.wf; import java.util.HashMap; +import java.util.List; import java.util.Objects; import java.util.UUID; import java.util.concurrent.Executors; @@ -24,14 +25,19 @@ import eu.dnetlib.is.resource.repository.SimpleResourceRepository; import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.procs.ProcessEngine; +import eu.dnetlib.manager.wf.workflows.procs.ProcessFactory; +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; +import eu.dnetlib.utils.CountedValue; import eu.dnetlib.utils.Stoppable; import eu.dnetlib.utils.StoppableDetails; @Service -public class WorkflowExecutor implements Stoppable { +public class WorkflowManagerService implements Stoppable { - private static final Log log = LogFactory.getLog(WorkflowExecutor.class); + private static final Log log = LogFactory.getLog(WorkflowManagerService.class); @Autowired private ProcessRegistry processRegistry; @@ -63,6 +69,10 @@ public class WorkflowExecutor implements Stoppable { }, 10, 10, TimeUnit.SECONDS); } + public WorkflowInstance findWorkflowInstance(final String id) throws WorkflowManagerException { + return workflowInstanceRepository.findById(id).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + id)); + } + public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, @@ -111,8 +121,8 @@ public class WorkflowExecutor implements Stoppable { throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown"); } - final WorkflowInstance instance = - workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId)); + final WorkflowInstance instance = findWorkflowInstance(wfInstanceId); + return startWorkflowInstance(instance, onSuccess, onFail); } @@ -179,4 +189,9 @@ public class WorkflowExecutor implements Stoppable { public void setPaused(final boolean paused) { this.paused = paused; } + + public List families() { + return workflowInstanceRepository.families(); + } + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 28f927f0..11c02092 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -12,10 +12,10 @@ import org.springframework.scheduling.support.CronExpression; import org.springframework.stereotype.Service; import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.wf.WorkflowManagerService; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; -import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; @Service @@ -24,7 +24,7 @@ public class ScheduledWorkflowLauncher { private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class); @Autowired - private WorkflowExecutor workflowExecutor; + private WorkflowManagerService wfManagerService; @Autowired private ProcessRegistry processRegistry; @@ -35,10 +35,10 @@ public class ScheduledWorkflowLauncher { @Autowired private WorkflowLogger logger; - @Value("${'dnet.workflow.scheduler.windowSize'}") + @Value("${dnet.workflow.scheduler.windowSize:1800000}") private int windowSize; // 1800000 are 30 minutes - @Scheduled(cron = "${'dnet.workflow.scheduler.cron'}") + @Scheduled(fixedRateString = "${dnet.workflow.scheduler.fixedRate:900000}") // 900000 are 5 minutes public void verifySheduledWorkflows() { log.debug("Verifying scheduled workflows - START"); @@ -51,7 +51,7 @@ public class ScheduledWorkflowLauncher { .filter(this::isReady) .forEach(instance -> { try { - workflowExecutor.startWorkflowInstance(instance, null, null); + wfManagerService.startWorkflowInstance(instance, null, null); } catch (final Exception e) { log.error("Error launching scheduled wf instance: " + instance.getId(), e); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index 663da422..0261f917 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -7,10 +7,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; +import eu.dnetlib.manager.wf.WorkflowManagerService; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; import eu.dnetlib.manager.wf.workflows.procs.Token; -import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; /** @@ -23,7 +23,7 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { private String wfId; @Autowired - private WorkflowExecutor executor; + private WorkflowManagerService wfManagerService; private WorkflowProcess process; @@ -49,7 +49,7 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { instance.setSystemParams(process.getGlobalParams()); instance.setUserParams(new HashMap<>()); - final String procId = executor.startWorkflowInstance(instance, (proc) -> { + final String procId = wfManagerService.startWorkflowInstance(instance, (proc) -> { log.debug("Child workflow has been completed successfully"); token.release(); }, (proc) -> { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java index fd1180b6..c9e0dbf2 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java @@ -25,7 +25,7 @@ public class ProcessRegistry { private final PriorityBlockingQueue pendingProcs = new PriorityBlockingQueue<>(); - @Value("${'dnet.wf.registry.size'}") + @Value("${dnet.wf.registry.size:100}") private int maxSize; synchronized public int countRunningWfs() { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java index 35cf72ce..f50ffc00 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java @@ -7,6 +7,7 @@ import org.springframework.beans.BeansException; import org.springframework.beans.PropertyAccessorFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; import eu.dnetlib.errors.WorkflowManagerException; import eu.dnetlib.manager.wf.nodes.DefaultJobNode; @@ -17,9 +18,7 @@ import eu.dnetlib.manager.wf.workflows.procs.Env; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; -/** - * Created by michele on 19/11/15. - */ +@Component public class NodeHelper implements ApplicationContextAware { public static final String beanNamePrefix = "wfNode"; From 836c31b6d61c28ed9388e4689844102b40eaf50c Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 20 Mar 2023 11:07:54 +0100 Subject: [PATCH 16/19] refactoring --- .../manager/wf/WorkflowManagerService.java | 17 +++++++-------- .../wf/cron/ScheduledWorkflowLauncher.java | 2 +- .../wf/nodes/LaunchWorkflowJobNode.java | 21 +++++++++++++------ .../wf/workflows/procs/ExecutionCallback.java | 9 ++++++++ .../wf/workflows/procs/ProcessEngine.java | 15 ++++++++++--- .../wf/workflows/procs/ProcessFactory.java | 6 ++---- .../manager/wf/workflows/procs/Token.java | 21 ++++++++----------- .../wf/workflows/procs/WorkflowProcess.java | 21 ++++++++----------- 8 files changed, 64 insertions(+), 48 deletions(-) create mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java index 82d7964c..f98d8af4 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -6,7 +6,6 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import javax.annotation.PostConstruct; @@ -25,6 +24,7 @@ import eu.dnetlib.is.resource.repository.SimpleResourceRepository; import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.procs.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.procs.ProcessEngine; import eu.dnetlib.manager.wf.workflows.procs.ProcessFactory; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; @@ -76,8 +76,7 @@ public class WorkflowManagerService implements Stoppable { public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, - final Consumer onSuccess, - final Consumer onFail) + final ExecutionCallback callback) throws WorkflowManagerException { if (isPaused()) { @@ -105,7 +104,7 @@ public class WorkflowManagerService implements Stoppable { instance.setSystemParams(new HashMap<>()); instance.setUserParams(new HashMap<>()); - return startWorkflowInstance(instance, onSuccess, onFail); + return startWorkflowInstance(instance, callback); } catch (final DsmException e) { throw new WorkflowManagerException("Invalid datasource: " + dsId, e); } @@ -113,8 +112,7 @@ public class WorkflowManagerService implements Stoppable { public String startWorkflowInstance(final String wfInstanceId, final String parent, - final Consumer onSuccess, - final Consumer onFail) throws Exception { + final ExecutionCallback callback) throws Exception { if (isPaused()) { log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown"); @@ -123,12 +121,11 @@ public class WorkflowManagerService implements Stoppable { final WorkflowInstance instance = findWorkflowInstance(wfInstanceId); - return startWorkflowInstance(instance, onSuccess, onFail); + return startWorkflowInstance(instance, callback); } public String startWorkflowInstance(final WorkflowInstance wfInstance, - final Consumer onSuccess, - final Consumer onFail) + final ExecutionCallback callback) throws WorkflowManagerException { if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) { @@ -153,7 +150,7 @@ public class WorkflowManagerService implements Stoppable { .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId())); final WorkflowProcess process = - processFactory.newProcess(wfMetadata, wfGraph, wfInstance, onSuccess, onFail); + processFactory.newProcess(wfMetadata, wfGraph, wfInstance, callback); return processRegistry.registerProcess(process, wfInstance); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java index 11c02092..2472c8a0 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -51,7 +51,7 @@ public class ScheduledWorkflowLauncher { .filter(this::isReady) .forEach(instance -> { try { - wfManagerService.startWorkflowInstance(instance, null, null); + wfManagerService.startWorkflowInstance(instance, null); } catch (final Exception e) { log.error("Error launching scheduled wf instance: " + instance.getId(), e); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index 0261f917..f169bbec 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Autowired; import eu.dnetlib.manager.wf.WorkflowManagerService; import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.procs.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; import eu.dnetlib.manager.wf.workflows.procs.Token; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; @@ -49,12 +50,20 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { instance.setSystemParams(process.getGlobalParams()); instance.setUserParams(new HashMap<>()); - final String procId = wfManagerService.startWorkflowInstance(instance, (proc) -> { - log.debug("Child workflow has been completed successfully"); - token.release(); - }, (proc) -> { - log.error("Child workflow is failed"); - token.releaseAsFailed("Child workflow is failed"); + final String procId = wfManagerService.startWorkflowInstance(instance, new ExecutionCallback() { + + @Override + public void onSuccess(final WorkflowProcess t) { + log.debug("Child workflow has been completed successfully"); + token.release(); + } + + @Override + public void onFail(final WorkflowProcess t) { + log.error("Child workflow is failed"); + token.releaseAsFailed("Child workflow is failed"); + } + }); if (log.isDebugEnabled()) { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java new file mode 100644 index 00000000..53cff2d0 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +public interface ExecutionCallback { + + void onSuccess(T t); + + void onFail(T t); + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java index 3755628a..edae89dc 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -106,9 +106,18 @@ public class ProcessEngine { } private Token prepareNewToken(final WorkflowProcess process, final GraphNode node) { - return new Token( - token -> releaseToken(process, node, token), - token -> completeProcess(process, token)); + return new Token(new ExecutionCallback() { + + @Override + public void onSuccess(final Token t) { + releaseToken(process, node, t); + } + + @Override + public void onFail(final Token t) { + completeProcess(process, t); + } + }); } private Map mergeEnvParams(final Token... tokens) { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index c700b1e2..bfa45344 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -4,7 +4,6 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; -import java.util.function.Consumer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,8 +32,7 @@ public class ProcessFactory { public WorkflowProcess newProcess(final SimpleResource wfMetadata, final WorkflowGraph wfGraph, final WorkflowInstance wfInstance, - final Consumer onSuccess, - final Consumer onFail) throws WorkflowManagerException { + final ExecutionCallback callback) throws WorkflowManagerException { final Map globalParams = new HashMap<>(); globalParams.putAll(wfInstance.getSystemParams()); @@ -42,7 +40,7 @@ public class ProcessFactory { final Graph graph = graphLoader.loadGraph(wfGraph, globalParams); - return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, onSuccess, onFail); + return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, callback); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java index c4635939..6d4c48de 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -2,7 +2,6 @@ package eu.dnetlib.manager.wf.workflows.procs; import java.time.LocalDateTime; import java.util.UUID; -import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; @@ -18,8 +17,7 @@ public class Token { private final LocalDateTime startDate; private LocalDateTime endDate = LocalDateTime.MIN; - private final Consumer onSuccess; - private final Consumer onFail; + private final ExecutionCallback callback; private boolean failed = false; @@ -27,11 +25,10 @@ public class Token { private String error = ""; private String errorStackTrace = ""; - public Token(final Consumer onSuccess, final Consumer onFail) { + public Token(final ExecutionCallback callback) { this.id = "token-" + UUID.randomUUID(); this.startDate = LocalDateTime.now(); - this.onSuccess = onSuccess; - this.onFail = onFail; + this.callback = callback; } public String getId() { @@ -73,8 +70,8 @@ public class Token { public void release() { setEndDate(LocalDateTime.now()); setActive(false); - if (this.onSuccess != null) { - this.onSuccess.accept(this); + if (this.callback != null) { + this.callback.onSuccess(this); } } @@ -84,8 +81,8 @@ public class Token { setFailed(true); setError(e.getMessage()); setErrorStackTrace(Throwables.getStackTraceAsString(e)); - if (this.onFail != null) { - this.onFail.accept(this); + if (this.callback != null) { + this.callback.onFail(this); } } @@ -94,8 +91,8 @@ public class Token { setActive(false); setFailed(true); setError(error); - if (this.onFail != null) { - this.onFail.accept(this); + if (this.callback != null) { + this.callback.onFail(this); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index 78807371..db5d8b69 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -6,7 +6,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; import org.apache.commons.lang3.math.NumberUtils; @@ -39,8 +38,7 @@ public class WorkflowProcess implements Comparable { private final SimpleResource wfMetadata; private final WorkflowInstance wfInstance; private final Graph graph; - private final Consumer onSuccess; - private final Consumer onFail; + private final ExecutionCallback callback; private final Env env; private final List tokens = new CopyOnWriteArrayList<>(); private LocalDateTime lastActivityDate; @@ -59,14 +57,12 @@ public class WorkflowProcess implements Comparable { final WorkflowInstance wfInstance, final Graph graph, final Map globalParams, - final Consumer onSuccess, - final Consumer onFail) { + final ExecutionCallback callback) { this.id = id; this.wfMetadata = wfMetadata; this.wfInstance = wfInstance; this.graph = graph; - this.onSuccess = onSuccess; - this.onFail = onFail; + this.callback = callback; this.status = Status.CREATED; this.env = new Env(); this.globalParams = globalParams; @@ -223,11 +219,12 @@ public class WorkflowProcess implements Comparable { setLastActivityDate(LocalDateTime.now()); } - if (token.isFailed() && onFail != null) { - onFail.accept(this); - } - if (!token.isFailed() && onSuccess != null) { - onSuccess.accept(this);; + if (callback != null) { + if (token.isFailed()) { + callback.onFail(this); + } else { + callback.onSuccess(this);; + } } } From ab3c0851ed3c7829e36a2f7818f5d4df78c80041 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 20 Mar 2023 12:26:40 +0100 Subject: [PATCH 17/19] refactoring --- .../manager/wf/WorkflowManagerService.java | 2 +- .../manager/wf/nodes/AbstractJobNode.java | 42 ++++++++++++++---- .../manager/wf/nodes/AsyncJobNode.java | 28 ------------ .../manager/wf/nodes/DefaultJobNode.java | 9 ++-- .../wf/nodes/LaunchWorkflowJobNode.java | 10 +++-- .../dnetlib/manager/wf/nodes/ProcessNode.java | 15 +------ .../manager/wf/nodes/SimpleJobNode.java | 7 +-- .../dnetlib/manager/wf/nodes/SuccessNode.java | 7 +-- .../wf/workflows/procs/ProcessEngine.java | 43 ++++++++++--------- .../wf/workflows/procs/ProcessFactory.java | 1 + .../manager/wf/workflows/procs/Token.java | 32 +++++++------- .../wf/workflows/procs/WorkflowProcess.java | 1 + .../{procs => util}/ExecutionCallback.java | 2 +- .../manager/wf/workflows/util/NodeHelper.java | 2 +- 14 files changed, 95 insertions(+), 106 deletions(-) delete mode 100644 libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java rename libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/{procs => util}/ExecutionCallback.java (64%) diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java index f98d8af4..85b88dd4 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -24,11 +24,11 @@ import eu.dnetlib.is.resource.repository.SimpleResourceRepository; import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; -import eu.dnetlib.manager.wf.workflows.procs.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.procs.ProcessEngine; import eu.dnetlib.manager.wf.workflows.procs.ProcessFactory; import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; import eu.dnetlib.utils.CountedValue; import eu.dnetlib.utils.Stoppable; diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java index 212fd659..21065752 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java @@ -1,34 +1,60 @@ package eu.dnetlib.manager.wf.nodes; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import eu.dnetlib.manager.wf.workflows.procs.Env; import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; public abstract class AbstractJobNode extends ProcessNode { - private static final Log log = LogFactory.getLog(SimpleJobNode.class); + private static final Log log = LogFactory.getLog(AbstractJobNode.class); - protected void doExecute(final Token token) { + private final boolean async; + + public AbstractJobNode(final boolean async) { + this.async = async; + } + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Override + public final void execute(final Token token, final ExecutionCallback callback) { try { log.debug("START NODE: " + getBeanName()); - setProgressMessage(getNodeName()); + token.setProgressMessage(getNodeName()); beforeStart(token); - execute(token.getEnv()); - beforeCompleted(token); + if (isAsync()) { + executor.execute(() -> doExecute(token, callback)); + } else { + doExecute(token, callback); + } log.debug("END NODE (SUCCESS): " + getBeanName()); } catch (final Throwable e) { log.error("got exception while executing workflow node", e); log.debug("END NODE (FAILED): " + getBeanName()); beforeFailed(token); - token.releaseAsFailed(e); + callback.onFail(token); } + } - abstract protected void execute(final Env env) throws Exception; + private final void doExecute(final Token token, final ExecutionCallback callback) { + execute(token); + beforeCompleted(token); + callback.onSuccess(token); + } + + protected abstract void execute(Token token); + + public final boolean isAsync() { + return async; + } protected void beforeStart(final Token token) { // For optional overwrites diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java deleted file mode 100644 index fe84b46d..00000000 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AsyncJobNode.java +++ /dev/null @@ -1,28 +0,0 @@ -package eu.dnetlib.manager.wf.nodes; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import eu.dnetlib.manager.wf.workflows.procs.Token; - -public abstract class AsyncJobNode extends AbstractJobNode { - - /** - * logger. - */ - private static final Log log = LogFactory.getLog(AsyncJobNode.class); - - private final ExecutorService executor = Executors.newCachedThreadPool(); - - @Override - public final void execute(final Token token) { - - log.info("executing async node"); - - executor.execute(() -> doExecute(token)); - } - -} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java index c3d97e53..ba6a54f8 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java @@ -5,15 +5,14 @@ import eu.dnetlib.manager.wf.workflows.procs.Token; /** * Created by michele on 26/11/15. */ -public final class DefaultJobNode extends ProcessNode { +public final class DefaultJobNode extends AbstractJobNode { public DefaultJobNode(final String name) { - super(); + super(false); setNodeName(name); } @Override - public void execute(final Token token) { - token.release(); - } + public void execute(final Token token) {} + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java index f169bbec..f05beb18 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -9,10 +9,10 @@ import org.springframework.beans.factory.annotation.Autowired; import eu.dnetlib.manager.wf.WorkflowManagerService; import eu.dnetlib.manager.wf.model.WorkflowInstance; -import eu.dnetlib.manager.wf.workflows.procs.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; import eu.dnetlib.manager.wf.workflows.procs.Token; import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; /** * Created by michele on 18/11/15. @@ -29,7 +29,7 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { private WorkflowProcess process; @Override - public final void execute(final Token token) { + public final void execute(final Token token, final ExecutionCallback callback) { try { final WorkflowInstance instance = new WorkflowInstance(); @@ -56,12 +56,14 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { public void onSuccess(final WorkflowProcess t) { log.debug("Child workflow has been completed successfully"); token.release(); + callback.onSuccess(token); } @Override public void onFail(final WorkflowProcess t) { log.error("Child workflow is failed"); token.releaseAsFailed("Child workflow is failed"); + callback.onFail(token); } }); @@ -70,11 +72,11 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { log.debug("The child workflow [instance: " + instance.getId() + "] is starting with procId: " + procId); } - setProgressMessage("Launched sub workflow, proc: " + procId); + token.setProgressMessage("Launched sub workflow, proc: " + procId); } catch (final Throwable e) { log.error("got exception while launching child workflow", e); - token.releaseAsFailed(e); + callback.onFail(token); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java index 48664307..f2d1c433 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java @@ -3,19 +3,15 @@ package eu.dnetlib.manager.wf.nodes; import org.springframework.beans.factory.BeanNameAware; import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; -/** - * Created by michele on 19/11/15. - */ public abstract class ProcessNode implements BeanNameAware { private String beanName; private String nodeName; - private String progressMessage; - - public abstract void execute(final Token token); + public abstract void execute(final Token token, ExecutionCallback callback); public String getBeanName() { return this.beanName; @@ -39,11 +35,4 @@ public abstract class ProcessNode implements BeanNameAware { return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName); } - public String getProgressMessage() { - return progressMessage; - } - - public void setProgressMessage(final String progressMessage) { - this.progressMessage = progressMessage; - } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java index 1dd14429..4339b5d1 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java @@ -1,12 +1,9 @@ package eu.dnetlib.manager.wf.nodes; -import eu.dnetlib.manager.wf.workflows.procs.Token; - public abstract class SimpleJobNode extends AbstractJobNode { - @Override - public final void execute(final Token token) { - doExecute(token); + public SimpleJobNode() { + super(false); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java index 022d3cec..c61c04be 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java @@ -5,13 +5,14 @@ import eu.dnetlib.manager.wf.workflows.procs.Token; /** * Created by michele on 26/11/15. */ -public class SuccessNode extends ProcessNode { +public class SuccessNode extends AbstractJobNode { public SuccessNode() { - super(); + super(false); setNodeName("success"); } @Override - public void execute(final Token token) {} + protected void execute(final Token token) {} + } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java index edae89dc..6585c39b 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -18,6 +18,7 @@ import eu.dnetlib.manager.history.WorkflowLogger; import eu.dnetlib.manager.wf.nodes.ProcessNode; import eu.dnetlib.manager.wf.notification.EmailSender; import eu.dnetlib.manager.wf.workflows.graph.GraphNode; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.util.NodeHelper; @Service @@ -45,11 +46,11 @@ public class ProcessEngine { try { for (final GraphNode node : process.getGraph().startNodes()) { final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv()); - final Token token = prepareNewToken(process, node); + final Token token = new Token(); token.getEnv().addAttributes(process.getEnv().getAttributes()); process.getTokens().add(token); - pNode.execute(token); + pNode.execute(token, newNodeCallback(process, node)); } } catch (final Throwable e) { log.error("WorkflowProcess node instantiation failed", e); @@ -57,6 +58,21 @@ public class ProcessEngine { } } + private ExecutionCallback newNodeCallback(final WorkflowProcess process, final GraphNode node) { + return new ExecutionCallback() { + + @Override + public void onSuccess(final Token t) { + releaseToken(process, node, t); + } + + @Override + public void onFail(final Token t) { + completeProcess(process, t); + } + }; + } + public void releaseToken(final WorkflowProcess process, final GraphNode oldGraphNode, final Token oldToken) { process.setLastActivityDate(LocalDateTime.now()); @@ -72,7 +88,7 @@ public class ProcessEngine { list.add(oldToken); if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) { - final Token token = prepareNewToken(process, node); + final Token token = new Token(); token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()]))); final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); @@ -82,17 +98,17 @@ public class ProcessEngine { if (node.isSucessNode()) { completeProcess(process, token); } else { - pNode.execute(token); + pNode.execute(token, newNodeCallback(process, node)); } } } else { - final Token token = prepareNewToken(process, node); + final Token token = new Token(); token.getEnv().addAttributes(oldToken.getEnv().getAttributes()); final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); process.getTokens().add(token); process.setLastActivityDate(LocalDateTime.now()); - pNode.execute(token); + pNode.execute(token, newNodeCallback(process, node)); } } } catch (final Throwable e) { @@ -105,21 +121,6 @@ public class ProcessEngine { } - private Token prepareNewToken(final WorkflowProcess process, final GraphNode node) { - return new Token(new ExecutionCallback() { - - @Override - public void onSuccess(final Token t) { - releaseToken(process, node, t); - } - - @Override - public void onFail(final Token t) { - completeProcess(process, t); - } - }); - } - private Map mergeEnvParams(final Token... tokens) { final Map map = new HashMap<>(); Arrays.stream(tokens).forEach(t -> map.putAll(t.getEnv().getAttributes())); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java index bfa45344..68c493bf 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -16,6 +16,7 @@ import eu.dnetlib.manager.wf.model.WorkflowGraph; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.Graph; import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; @Component public class ProcessFactory { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java index 6d4c48de..bfb1541e 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -14,21 +14,22 @@ public class Token { private final String id; private final Env env = new Env(); + + private String progressMessage; private final LocalDateTime startDate; - private LocalDateTime endDate = LocalDateTime.MIN; + private LocalDateTime endDate; - private final ExecutionCallback callback; + private boolean failed; + private boolean active; - private boolean failed = false; - - private boolean active = true; private String error = ""; private String errorStackTrace = ""; - public Token(final ExecutionCallback callback) { + public Token() { this.id = "token-" + UUID.randomUUID(); this.startDate = LocalDateTime.now(); - this.callback = callback; + this.failed = false; + this.active = true; } public String getId() { @@ -70,9 +71,6 @@ public class Token { public void release() { setEndDate(LocalDateTime.now()); setActive(false); - if (this.callback != null) { - this.callback.onSuccess(this); - } } public void releaseAsFailed(final Throwable e) { @@ -81,9 +79,6 @@ public class Token { setFailed(true); setError(e.getMessage()); setErrorStackTrace(Throwables.getStackTraceAsString(e)); - if (this.callback != null) { - this.callback.onFail(this); - } } public void releaseAsFailed(final String error) { @@ -91,9 +86,6 @@ public class Token { setActive(false); setFailed(true); setError(error); - if (this.callback != null) { - this.callback.onFail(this); - } } public void checkStatus() { @@ -106,6 +98,14 @@ public class Token { } } + public String getProgressMessage() { + return progressMessage; + } + + public void setProgressMessage(final String progressMessage) { + this.progressMessage = progressMessage; + } + public String getError() { return this.error; } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index db5d8b69..0bb36211 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -13,6 +13,7 @@ import eu.dnetlib.is.model.resource.SimpleResource; import eu.dnetlib.manager.history.model.WfProcessExecution; import eu.dnetlib.manager.wf.model.WorkflowInstance; import eu.dnetlib.manager.wf.workflows.graph.Graph; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; /** diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java similarity index 64% rename from libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java rename to libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java index 53cff2d0..04bcdf38 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ExecutionCallback.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java @@ -1,4 +1,4 @@ -package eu.dnetlib.manager.wf.workflows.procs; +package eu.dnetlib.manager.wf.workflows.util; public interface ExecutionCallback { diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java index f50ffc00..c9c11307 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java @@ -10,8 +10,8 @@ import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; import eu.dnetlib.errors.WorkflowManagerException; -import eu.dnetlib.manager.wf.nodes.DefaultJobNode; import eu.dnetlib.manager.wf.nodes.ProcessNode; +import eu.dnetlib.manager.wf.nodes.DefaultJobNode; import eu.dnetlib.manager.wf.nodes.SuccessNode; import eu.dnetlib.manager.wf.workflows.graph.GraphNode; import eu.dnetlib.manager.wf.workflows.procs.Env; From 28598ddfea6affad53995cd1c3f8add7c00ee772 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 20 Mar 2023 15:32:46 +0100 Subject: [PATCH 18/19] expression parser --- .../wf/workflows/graph/GraphLoader.java | 14 ++++- .../wf/workflows/graph/GraphLoaderTest.java | 54 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java index 0b4c23e9..ff9cfa29 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -6,7 +6,11 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.expression.MapAccessor; import org.springframework.core.env.Environment; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; import org.springframework.stereotype.Service; import com.google.common.base.Function; @@ -64,8 +68,14 @@ public class GraphLoader { } private Function generateFunction(final String condition) { - // TODO Auto-generated method stub - return null; + return env -> { + final ExpressionParser parser = new SpelExpressionParser(); + + final StandardEvaluationContext context = new StandardEvaluationContext(env.getAttributes()); + context.addPropertyAccessor(new MapAccessor()); + + return parser.parseExpression(condition).getValue(context, Boolean.class); + }; } private void checkValidity(final Graph graph) throws WorkflowManagerException { diff --git a/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java b/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java new file mode 100644 index 00000000..f76b610b --- /dev/null +++ b/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java @@ -0,0 +1,54 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.function.Function; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.expression.MapAccessor; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class GraphLoaderTest { + + private Env env; + + @BeforeEach + void setUp() throws Exception { + env = new Env(); + env.setAttribute("author", "Michele Artini"); + env.setAttribute("age", 47); + } + + @Test + final void testExpressions() { + assertTrue(evalFunction("age == 47").apply(env)); + assertTrue(evalFunction("age > 40").apply(env)); + assertTrue(evalFunction("author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 && author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 || author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 || author == 'Claudio Atzori'").apply(env)); + assertTrue(evalFunction("age == 22 || author == 'Michele Artini'").apply(env)); + assertFalse(evalFunction("age != 47").apply(env)); + assertFalse(evalFunction("age < 40").apply(env)); + assertFalse(evalFunction("author != 'Michele Artini'").apply(env)); + assertFalse(evalFunction("age == 47 && author == 'Claudio Atzori'").apply(env)); + } + + private Function evalFunction(final String f) { + return env -> { + final ExpressionParser parser = new SpelExpressionParser(); + + final StandardEvaluationContext context = new StandardEvaluationContext(env.getAttributes()); + context.addPropertyAccessor(new MapAccessor()); + + return parser.parseExpression(f).getValue(context, Boolean.class); + }; + } + +} From 806f8e9227661406319afe8ba8639fe666d891d0 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 21 Mar 2023 12:13:23 +0100 Subject: [PATCH 19/19] ajax methods for wfs --- .../eu/dnetlib/dsm/DsmAjaxController.java | 4 +-- .../dnetlib/is/info/InfoAjaxController.java | 36 +++++++++---------- .../java/eu/dnetlib/is/info/KeyValue.java | 8 ++--- .../manager/wf/WfInstancesController.java | 19 +++++++--- .../dnet-is-application/src/app/app.module.ts | 4 ++- .../src/app/common/is.service.ts | 21 +++++++++++ .../wf-instances/wf-instances.component.html | 8 ++++- .../manager/wf/model/WorkflowInstance.java | 22 ++++++++++++ .../src/main/resources/sql/schema.sql | 2 ++ .../WorkflowInstanceRepository.java | 12 ++++--- .../manager/wf/WorkflowManagerService.java | 10 ++++-- .../wf/workflows/procs/WorkflowProcess.java | 2 +- 12 files changed, 109 insertions(+), 39 deletions(-) diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java index 330d8e95..a9d49ea1 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java @@ -40,9 +40,9 @@ public class DsmAjaxController extends AbstractDnetController { private ProtocolService protocolService; @GetMapping("/browsableFields") - public List browsableFields() { + public List> browsableFields() { return Arrays.stream(DsmBrowsableFields.values()) - .map(f -> new KeyValue(f.name(), f.desc)) + .map(f -> new KeyValue<>(f.name(), f.desc)) .collect(Collectors.toList()); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java index 8b98de3c..6b0f2b8b 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java @@ -56,30 +56,30 @@ public class InfoAjaxController extends AbstractDnetController { return res; } - private InfoSection jvm() { - final InfoSection jvm = new InfoSection<>("JVM"); - jvm.getData().add(new KeyValue("JVM Name", mxbean.getVmName())); - jvm.getData().add(new KeyValue("JVM Vendor", mxbean.getVmVendor())); - jvm.getData().add(new KeyValue("JVM Version", mxbean.getVmVersion())); - jvm.getData().add(new KeyValue("JVM Spec Name", mxbean.getSpecName())); - jvm.getData().add(new KeyValue("JVM Spec Vendor", mxbean.getSpecVendor())); - jvm.getData().add(new KeyValue("JVM Spec Version", mxbean.getSpecVersion())); - jvm.getData().add(new KeyValue("Running JVM Name", mxbean.getName())); - jvm.getData().add(new KeyValue("Management Spec Version", mxbean.getManagementSpecVersion())); + private InfoSection> jvm() { + final InfoSection> jvm = new InfoSection<>("JVM"); + jvm.getData().add(new KeyValue<>("JVM Name", mxbean.getVmName())); + jvm.getData().add(new KeyValue<>("JVM Vendor", mxbean.getVmVendor())); + jvm.getData().add(new KeyValue<>("JVM Version", mxbean.getVmVersion())); + jvm.getData().add(new KeyValue<>("JVM Spec Name", mxbean.getSpecName())); + jvm.getData().add(new KeyValue<>("JVM Spec Vendor", mxbean.getSpecVendor())); + jvm.getData().add(new KeyValue<>("JVM Spec Version", mxbean.getSpecVersion())); + jvm.getData().add(new KeyValue<>("Running JVM Name", mxbean.getName())); + jvm.getData().add(new KeyValue<>("Management Spec Version", mxbean.getManagementSpecVersion())); return jvm; } - private InfoSection args() { - final InfoSection libs = new InfoSection<>("Arguments"); - libs.getData().add(new KeyValue("Input arguments", StringUtils.join(mxbean.getInputArguments(), " "))); + private InfoSection> args() { + final InfoSection> libs = new InfoSection<>("Arguments"); + libs.getData().add(new KeyValue<>("Input arguments", StringUtils.join(mxbean.getInputArguments(), " "))); return libs; } - private List> props() { - final List> res = new ArrayList<>(); + private List>> props() { + final List>> res = new ArrayList<>(); configurableEnvironment.getPropertySources().forEach(ps -> { - final InfoSection section = new InfoSection<>("Properties: " + ps.getName()); + final InfoSection> section = new InfoSection<>("Properties: " + ps.getName()); addAllProperties(section, ps); res.add(section); }); @@ -87,13 +87,13 @@ public class InfoAjaxController extends AbstractDnetController { return res; } - private void addAllProperties(final InfoSection res, final PropertySource ps) { + private void addAllProperties(final InfoSection> res, final PropertySource ps) { if (ps instanceof CompositePropertySource) { final CompositePropertySource cps = (CompositePropertySource) ps; cps.getPropertySources().forEach(x -> addAllProperties(res, x)); } else if (ps instanceof EnumerablePropertySource) { final EnumerablePropertySource eps = (EnumerablePropertySource) ps; - Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue(k, eps.getProperty(k)))); + Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue<>(k, eps.getProperty(k)))); } else {} } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java index 082e63fb..e7abe9f9 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java @@ -1,11 +1,11 @@ package eu.dnetlib.is.info; -public class KeyValue { +public class KeyValue { private final String k; - private final Object v; + private final T v; - public KeyValue(final String k, final Object v) { + public KeyValue(final String k, final T v) { this.k = k; this.v = v; } @@ -14,7 +14,7 @@ public class KeyValue { return k; } - public Object getV() { + public T getV() { return v; } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java index d8fc4096..23e22ad5 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java @@ -1,15 +1,17 @@ package eu.dnetlib.manager.wf; import java.util.List; +import java.util.stream.Collectors; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.is.info.KeyValue; import eu.dnetlib.manager.wf.model.WorkflowInstance; -import eu.dnetlib.utils.CountedValue; @RestController @RequestMapping("/ajax/wf_instances") @@ -22,9 +24,18 @@ public class WfInstancesController extends AbstractDnetController { return wfManagerService.findWorkflowInstance(id); } - @GetMapping("/families") - public List listWfFamilies() throws Exception { - return wfManagerService.families(); + @GetMapping("/search") + public List> listWfInstances(@RequestParam final String section) throws Exception { + return wfManagerService.streamWfInstancesBySection(section) + .map(x -> new KeyValue<>(x.getId(), x.getName())) + .collect(Collectors.toList()); + } + + @GetMapping("/sections") + public List> listWfFamilies() throws Exception { + return wfManagerService.streamSections() + .map(x -> new KeyValue<>(x.getValue(), x.getCount())) + .collect(Collectors.toList()); } } diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 374a19b0..59ad6a0c 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -38,6 +38,7 @@ import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, Ad import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; import { EmailDialog, EmailsComponent } from './emails/emails.component'; import { WfInstancesComponent } from './wf-instances/wf-instances.component'; +import { MatTabsModule } from '@angular/material/tabs'; @NgModule({ declarations: [ @@ -100,7 +101,8 @@ import { WfInstancesComponent } from './wf-instances/wf-instances.component'; ReactiveFormsModule, MatSnackBarModule, MatPaginatorModule, - MatProgressSpinnerModule + MatProgressSpinnerModule, + MatTabsModule ], providers: [{ provide: HTTP_INTERCEPTORS, diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index 339da869..80fa5a04 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -350,6 +350,27 @@ export class ISService { }); } + loadWfIntancesSections(onSuccess: Function): void { + this.client.get('./ajax/wf_instances/sections').subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + loadWfIntances(section: string, onSuccess: Function): void { + this.client.get('./ajax/wf_instances/search?section=' + encodeURIComponent(section)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + loadWfIntance(id: string, onSuccess: Function): void { + this.client.get('./ajax/wf_instances/instance/' + encodeURIComponent(id)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + private showError(error: any, form?: FormGroup) { console.log(error); diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html index 87d7cd4b..2efe0381 100644 --- a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html +++ b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html @@ -1 +1,7 @@ -

wfs works!

+

Workflow Instances

+ + + Content 1 + Content 2 + Content 3 + diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java index 0b6ff317..d9478332 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -31,6 +31,12 @@ public class WorkflowInstance implements Serializable { @Column(name = "id") private String id; + @Column(name = "name") + private String name; + + @Column(name = "section") + private String section; + @Type(type = "jsonb") @Column(name = "details", columnDefinition = "jsonb") private Map details = new LinkedHashMap<>(); @@ -87,6 +93,22 @@ public class WorkflowInstance implements Serializable { this.id = id; } + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getSection() { + return section; + } + + public void setSection(final String section) { + this.section = section; + } + public Map getDetails() { return details; } diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index 2952dc26..041c22ab 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -264,6 +264,8 @@ CREATE TABLE emails ( CREATE TABLE workflow_instances ( id text PRIMARY KEY, + name text NOT NULL, + section text, details jsonb NOT NULL DEFAULT '{}', priority int, dsid text, diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java index e30934d5..1bab126e 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java @@ -1,6 +1,6 @@ package eu.dnetlib.manager.wf.repository; -import java.util.List; +import java.util.stream.Stream; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; @@ -10,10 +10,12 @@ import eu.dnetlib.utils.CountedValue; public interface WorkflowInstanceRepository extends JpaRepository { - @Query(value = "select r.subtype as value, count(*) as count " - + "from workflow_instances i join resources r on (i.workflow = r.id) " - + "group by r.subtype " + @Query(value = "select section as value, count(*) as count " + + "from workflow_instances " + + "group by section " + "order by count desc;", nativeQuery = true) - List families(); + Stream streamSections(); + + Stream findBySection(String section); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java index 85b88dd4..145b9949 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -1,11 +1,11 @@ package eu.dnetlib.manager.wf; import java.util.HashMap; -import java.util.List; import java.util.Objects; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import javax.annotation.PostConstruct; @@ -187,8 +187,12 @@ public class WorkflowManagerService implements Stoppable { this.paused = paused; } - public List families() { - return workflowInstanceRepository.families(); + public Stream streamSections() { + return workflowInstanceRepository.streamSections(); + } + + public Stream streamWfInstancesBySection(final String section) { + return workflowInstanceRepository.findBySection(section); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java index 0bb36211..caf32529 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -75,7 +75,7 @@ public class WorkflowProcess implements Comparable { } public String getName() { - return wfMetadata.getName(); + return wfInstance.getName(); } public String getFamily() {