refactoring
This commit is contained in:
parent
30991cabbf
commit
cd5e1ab957
|
@ -40,8 +40,8 @@ public class WorkflowLogger {
|
||||||
return wfProcessExecutionRepository.findById(processId).get();
|
return wfProcessExecutionRepository.findById(processId).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveProcessExecution(final WfProcessExecution proc) {
|
public void saveProcessExecution(final WfProcessExecution pe) {
|
||||||
wfProcessExecutionRepository.save(proc);
|
wfProcessExecutionRepository.save(pe);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Optional<WfProcessExecution> getLastExecutionForInstance(final String id) {
|
public Optional<WfProcessExecution> getLastExecutionForInstance(final String id) {
|
||||||
|
|
|
@ -17,15 +17,12 @@ import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
|
import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor;
|
import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
|
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
|
||||||
import eu.dnetlib.utils.DateUtils;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class ScheduledWorkflowLauncher {
|
public class ScheduledWorkflowLauncher {
|
||||||
|
|
||||||
private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class);
|
private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class);
|
||||||
|
|
||||||
private static final DateUtils dateUtils = new DateUtils();
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private WorkflowExecutor workflowExecutor;
|
private WorkflowExecutor workflowExecutor;
|
||||||
|
|
||||||
|
@ -54,7 +51,7 @@ public class ScheduledWorkflowLauncher {
|
||||||
.filter(this::isReady)
|
.filter(this::isReady)
|
||||||
.forEach(instance -> {
|
.forEach(instance -> {
|
||||||
try {
|
try {
|
||||||
workflowExecutor.startWorkflowInstance(instance, null, null);
|
workflowExecutor.startWorkflowInstance(instance, null, null, null);
|
||||||
} catch (final Exception e) {
|
} catch (final Exception e) {
|
||||||
log.error("Error launching scheduled wf instance: " + instance.getId(), e);
|
log.error("Error launching scheduled wf instance: " + instance.getId(), e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,9 +13,12 @@ public abstract class AbstractJobNode extends ProcessNode {
|
||||||
protected void doExecute(final Token token) {
|
protected void doExecute(final Token token) {
|
||||||
try {
|
try {
|
||||||
log.debug("START NODE: " + getBeanName());
|
log.debug("START NODE: " + getBeanName());
|
||||||
|
setProgressMessage(getNodeName());
|
||||||
|
|
||||||
beforeStart(token);
|
beforeStart(token);
|
||||||
execute(token.getEnv());
|
execute(token.getEnv());
|
||||||
beforeCompleted(token);
|
beforeCompleted(token);
|
||||||
|
|
||||||
log.debug("END NODE (SUCCESS): " + getBeanName());
|
log.debug("END NODE (SUCCESS): " + getBeanName());
|
||||||
} catch (final Throwable e) {
|
} catch (final Throwable e) {
|
||||||
log.error("got exception while executing workflow node", e);
|
log.error("got exception while executing workflow node", e);
|
||||||
|
|
|
@ -9,8 +9,6 @@ import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.Token;
|
import eu.dnetlib.manager.wf.workflows.procs.Token;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor;
|
import eu.dnetlib.manager.wf.workflows.procs.WorkflowExecutor;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
|
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
|
||||||
import eu.dnetlib.manager.wf.workflows.util.ProcessCallback;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.SubWorkflowProgressProvider;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Created by michele on 18/11/15.
|
* Created by michele on 18/11/15.
|
||||||
|
@ -33,28 +31,23 @@ public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware {
|
||||||
public final void execute(final Token token) {
|
public final void execute(final Token token) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
final String procId = executor.startWorkflowInstance(getWfInstanceId(), new ProcessCallback() {
|
final String procId = executor.startWorkflowInstance(getWfInstanceId(), process.getWfInstanceId(), (proc) -> {
|
||||||
|
log.debug("Child workflow has been completed successfully");
|
||||||
@Override
|
token.release();
|
||||||
public void onSuccess() {
|
}, (proc) -> {
|
||||||
log.debug("Child workflow has been completed successfully");
|
log.error("Child workflow is failed");
|
||||||
token.release();
|
token.releaseAsFailed("Child workflow is failed");
|
||||||
}
|
});
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onFail() {
|
|
||||||
log.error("Child workflow is failed");
|
|
||||||
token.releaseAsFailed("Child workflow is failed");
|
|
||||||
}
|
|
||||||
}, process.getWfInstanceId());
|
|
||||||
|
|
||||||
if (log.isDebugEnabled()) {
|
if (log.isDebugEnabled()) {
|
||||||
log.debug("The child workflow [instance: " + getWfInstanceId() + "] is starting with procId: " + procId);
|
log.debug("The child workflow [instance: " + getWfInstanceId() + "] is starting with procId: " + procId);
|
||||||
}
|
}
|
||||||
|
|
||||||
token.setProgressProvider(new SubWorkflowProgressProvider(procId, processRegistry));
|
setProgressMessage("Launched sub workflow, proc: " + procId);
|
||||||
|
|
||||||
} catch (final Throwable e) {
|
} catch (
|
||||||
|
|
||||||
|
final Throwable e) {
|
||||||
log.error("got exception while launching child workflow", e);
|
log.error("got exception while launching child workflow", e);
|
||||||
token.releaseAsFailed(e);
|
token.releaseAsFailed(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,6 +13,8 @@ public abstract class ProcessNode implements BeanNameAware {
|
||||||
|
|
||||||
private String nodeName;
|
private String nodeName;
|
||||||
|
|
||||||
|
private String progressMessage;
|
||||||
|
|
||||||
public abstract void execute(final Token token);
|
public abstract void execute(final Token token);
|
||||||
|
|
||||||
public String getBeanName() {
|
public String getBeanName() {
|
||||||
|
@ -36,4 +38,12 @@ public abstract class ProcessNode implements BeanNameAware {
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName);
|
return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getProgressMessage() {
|
||||||
|
return progressMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProgressMessage(final String progressMessage) {
|
||||||
|
this.progressMessage = progressMessage;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,11 +4,9 @@ import java.time.LocalDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
@ -17,14 +15,10 @@ import org.springframework.stereotype.Service;
|
||||||
import com.google.common.base.Throwables;
|
import com.google.common.base.Throwables;
|
||||||
|
|
||||||
import eu.dnetlib.manager.history.WorkflowLogger;
|
import eu.dnetlib.manager.history.WorkflowLogger;
|
||||||
import eu.dnetlib.manager.history.model.WfProcessExecution;
|
|
||||||
import eu.dnetlib.manager.wf.nodes.ProcessNode;
|
import eu.dnetlib.manager.wf.nodes.ProcessNode;
|
||||||
import eu.dnetlib.manager.wf.notification.EmailSender;
|
import eu.dnetlib.manager.wf.notification.EmailSender;
|
||||||
import eu.dnetlib.manager.wf.workflows.graph.GraphNode;
|
import eu.dnetlib.manager.wf.workflows.graph.GraphNode;
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.NodeHelper;
|
import eu.dnetlib.manager.wf.workflows.util.NodeHelper;
|
||||||
import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class ProcessEngine {
|
public class ProcessEngine {
|
||||||
|
@ -51,8 +45,7 @@ public class ProcessEngine {
|
||||||
try {
|
try {
|
||||||
for (final GraphNode node : process.getGraph().startNodes()) {
|
for (final GraphNode node : process.getGraph().startNodes()) {
|
||||||
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv());
|
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv());
|
||||||
final Token token = new Token(node.getName(), newNodeTokenCallback(process, node));
|
final Token token = prepareNewToken(process, node);
|
||||||
|
|
||||||
token.getEnv().addAttributes(process.getEnv().getAttributes());
|
token.getEnv().addAttributes(process.getEnv().getAttributes());
|
||||||
process.getTokens().add(token);
|
process.getTokens().add(token);
|
||||||
|
|
||||||
|
@ -79,7 +72,7 @@ public class ProcessEngine {
|
||||||
list.add(oldToken);
|
list.add(oldToken);
|
||||||
|
|
||||||
if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) {
|
if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) {
|
||||||
final Token token = new Token(node.getName(), newNodeTokenCallback(process, node));
|
final Token token = prepareNewToken(process, node);
|
||||||
token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()])));
|
token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()])));
|
||||||
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
|
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
|
||||||
|
|
||||||
|
@ -87,13 +80,13 @@ public class ProcessEngine {
|
||||||
process.setLastActivityDate(LocalDateTime.now());
|
process.setLastActivityDate(LocalDateTime.now());
|
||||||
|
|
||||||
if (node.isSucessNode()) {
|
if (node.isSucessNode()) {
|
||||||
markAsCompleted(process, token);
|
completeProcess(process, token);
|
||||||
} else {
|
} else {
|
||||||
pNode.execute(token);
|
pNode.execute(token);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
final Token token = new Token(node.getName(), newNodeTokenCallback(process, node));
|
final Token token = prepareNewToken(process, node);
|
||||||
token.getEnv().addAttributes(oldToken.getEnv().getAttributes());
|
token.getEnv().addAttributes(oldToken.getEnv().getAttributes());
|
||||||
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
|
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
|
||||||
|
|
||||||
|
@ -112,19 +105,10 @@ public class ProcessEngine {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private NodeTokenCallback newNodeTokenCallback(final WorkflowProcess process, final GraphNode node) {
|
private Token prepareNewToken(final WorkflowProcess process, final GraphNode node) {
|
||||||
return new NodeTokenCallback() {
|
return new Token(
|
||||||
|
token -> releaseToken(process, node, token),
|
||||||
@Override
|
token -> completeProcess(process, token));
|
||||||
public void onSuccess(final Token token) {
|
|
||||||
releaseToken(process, node, token);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onFail(final Token token) {
|
|
||||||
completeProcess(process, token);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, Object> mergeEnvParams(final Token... tokens) {
|
private Map<String, Object> mergeEnvParams(final Token... tokens) {
|
||||||
|
@ -133,71 +117,11 @@ public class ProcessEngine {
|
||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markAsCompleted(final WorkflowProcess process, final Token token) {
|
|
||||||
completeProcess(process, token);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void completeProcess(final WorkflowProcess process, final Token token) {
|
private void completeProcess(final WorkflowProcess process, final Token token) {
|
||||||
if (token.isActive()) {
|
token.checkStatus();
|
||||||
if (StringUtils.isNotBlank(token.getError())) {
|
process.complete(token);
|
||||||
token.releaseAsFailed(token.getError());
|
wfLogger.saveProcessExecution(process.asLog());
|
||||||
} else {
|
|
||||||
token.release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final LocalDateTime now = token.getEndDate();
|
|
||||||
|
|
||||||
process.setLastActivityDate(now);
|
|
||||||
process.setEndDate(now);
|
|
||||||
process.setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS);
|
|
||||||
|
|
||||||
if (token.isFailed()) {
|
|
||||||
process.setStatus(Status.FAILURE);
|
|
||||||
process.setError(token.getError());
|
|
||||||
process.setErrorStacktrace(token.getErrorStackTrace());
|
|
||||||
process.setLastActivityDate(LocalDateTime.now());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (process.getCallback() != null) {
|
|
||||||
if (token.isFailed()) {
|
|
||||||
process.getCallback().onFail();
|
|
||||||
} else {
|
|
||||||
process.getCallback().onSuccess();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final Map<String, String> details = new LinkedHashMap<>();
|
|
||||||
details.putAll(process.getOutputParams());
|
|
||||||
details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + process.getPriority());
|
|
||||||
details.put(WorkflowsConstants.LOG_WF_ID, process.getWfId());
|
|
||||||
details.put(WorkflowsConstants.LOG_WF_ID, process.getWfInstanceId());
|
|
||||||
|
|
||||||
if (process.getError() != null) {
|
|
||||||
details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, process.getError());
|
|
||||||
details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, process.getErrorStacktrace());
|
|
||||||
}
|
|
||||||
|
|
||||||
final WfProcessExecution pe = new WfProcessExecution();
|
|
||||||
pe.setProcessId(process.getId());
|
|
||||||
pe.setName(process.getName());
|
|
||||||
pe.setFamily(process.getFamily());
|
|
||||||
|
|
||||||
pe.setDsId(process.getDsId());
|
|
||||||
pe.setDsName(process.getDsName());
|
|
||||||
pe.setDsApi(process.getDsInterface());
|
|
||||||
|
|
||||||
pe.setStartDate(process.getStartDate());
|
|
||||||
pe.setEndDate(process.getEndDate());
|
|
||||||
|
|
||||||
pe.setStatus(process.getStatus().toString());
|
|
||||||
|
|
||||||
pe.setDetails(details);
|
|
||||||
|
|
||||||
wfLogger.saveProcessExecution(pe);
|
|
||||||
|
|
||||||
emailSender.sendMails(process);
|
emailSender.sendMails(process);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -16,7 +17,6 @@ import eu.dnetlib.manager.wf.model.WorkflowGraph;
|
||||||
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
||||||
import eu.dnetlib.manager.wf.workflows.graph.Graph;
|
import eu.dnetlib.manager.wf.workflows.graph.Graph;
|
||||||
import eu.dnetlib.manager.wf.workflows.graph.GraphLoader;
|
import eu.dnetlib.manager.wf.workflows.graph.GraphLoader;
|
||||||
import eu.dnetlib.manager.wf.workflows.util.ProcessCallback;
|
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class ProcessFactory {
|
public class ProcessFactory {
|
||||||
|
@ -33,8 +33,9 @@ public class ProcessFactory {
|
||||||
public WorkflowProcess newProcess(final SimpleResource wfMetadata,
|
public WorkflowProcess newProcess(final SimpleResource wfMetadata,
|
||||||
final WorkflowGraph wfGraph,
|
final WorkflowGraph wfGraph,
|
||||||
final WorkflowInstance wfInstance,
|
final WorkflowInstance wfInstance,
|
||||||
final ProcessCallback processCallback,
|
final String parent,
|
||||||
final String parent) throws WorkflowManagerException {
|
final Consumer<WorkflowProcess> onSuccess,
|
||||||
|
final Consumer<WorkflowProcess> onFail) throws WorkflowManagerException {
|
||||||
|
|
||||||
final Map<String, String> globalParams = new HashMap<>();
|
final Map<String, String> globalParams = new HashMap<>();
|
||||||
globalParams.putAll(wfInstance.getSystemParams());
|
globalParams.putAll(wfInstance.getSystemParams());
|
||||||
|
@ -42,7 +43,7 @@ public class ProcessFactory {
|
||||||
|
|
||||||
final Graph graph = graphLoader.loadGraph(wfGraph, globalParams);
|
final Graph graph = graphLoader.loadGraph(wfGraph, globalParams);
|
||||||
|
|
||||||
return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, processCallback, parent);
|
return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, parent, onSuccess, onFail);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,34 +2,36 @@ package eu.dnetlib.manager.wf.workflows.procs;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import com.google.common.base.Throwables;
|
import com.google.common.base.Throwables;
|
||||||
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.NodeTokenCallback;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.ProgressProvider;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Created by michele on 19/11/15.
|
* Created by michele on 19/11/15.
|
||||||
*/
|
*/
|
||||||
public class Token {
|
public class Token {
|
||||||
|
|
||||||
private final String id;
|
private final String id;
|
||||||
private final String nodeName;
|
|
||||||
private final Env env = new Env();
|
private final Env env = new Env();
|
||||||
private final LocalDateTime startDate;
|
private final LocalDateTime startDate;
|
||||||
private final NodeTokenCallback callback;
|
|
||||||
private boolean failed = false;
|
|
||||||
private LocalDateTime endDate = LocalDateTime.MIN;
|
private LocalDateTime endDate = LocalDateTime.MIN;
|
||||||
|
|
||||||
|
private final Consumer<Token> onSuccess;
|
||||||
|
private final Consumer<Token> onFail;
|
||||||
|
|
||||||
|
private boolean failed = false;
|
||||||
|
|
||||||
private boolean active = true;
|
private boolean active = true;
|
||||||
private String error = "";
|
private String error = "";
|
||||||
private String errorStackTrace = "";
|
private String errorStackTrace = "";
|
||||||
private ProgressProvider progressProvider;
|
|
||||||
|
|
||||||
public Token(final String nodeName, final NodeTokenCallback callback) {
|
public Token(final Consumer<Token> onSuccess, final Consumer<Token> onFail) {
|
||||||
this.id = "token-" + UUID.randomUUID();
|
this.id = "token-" + UUID.randomUUID();
|
||||||
this.nodeName = nodeName;
|
|
||||||
this.startDate = LocalDateTime.now();
|
this.startDate = LocalDateTime.now();
|
||||||
this.callback = callback;
|
this.onSuccess = onSuccess;
|
||||||
|
this.onFail = onFail;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getId() {
|
public String getId() {
|
||||||
|
@ -71,8 +73,8 @@ public class Token {
|
||||||
public void release() {
|
public void release() {
|
||||||
setEndDate(LocalDateTime.now());
|
setEndDate(LocalDateTime.now());
|
||||||
setActive(false);
|
setActive(false);
|
||||||
if (this.callback != null) {
|
if (this.onSuccess != null) {
|
||||||
this.callback.onSuccess(this);
|
this.onSuccess.accept(this);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,8 +84,8 @@ public class Token {
|
||||||
setFailed(true);
|
setFailed(true);
|
||||||
setError(e.getMessage());
|
setError(e.getMessage());
|
||||||
setErrorStackTrace(Throwables.getStackTraceAsString(e));
|
setErrorStackTrace(Throwables.getStackTraceAsString(e));
|
||||||
if (this.callback != null) {
|
if (this.onFail != null) {
|
||||||
this.callback.onFail(this);
|
this.onFail.accept(this);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -92,13 +94,19 @@ public class Token {
|
||||||
setActive(false);
|
setActive(false);
|
||||||
setFailed(true);
|
setFailed(true);
|
||||||
setError(error);
|
setError(error);
|
||||||
if (this.callback != null) {
|
if (this.onFail != null) {
|
||||||
this.callback.onFail(this);
|
this.onFail.accept(this);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getNodeName() {
|
public void checkStatus() {
|
||||||
return this.nodeName;
|
if (isActive()) {
|
||||||
|
if (StringUtils.isNotBlank(error)) {
|
||||||
|
releaseAsFailed(error);
|
||||||
|
} else {
|
||||||
|
release();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getError() {
|
public String getError() {
|
||||||
|
@ -117,12 +125,4 @@ public class Token {
|
||||||
this.errorStackTrace = errorStackTrace;
|
this.errorStackTrace = errorStackTrace;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ProgressProvider getProgressProvider() {
|
|
||||||
return this.progressProvider;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setProgressProvider(final ProgressProvider progressProvider) {
|
|
||||||
this.progressProvider = progressProvider;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@ import java.util.Objects;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import javax.annotation.PostConstruct;
|
import javax.annotation.PostConstruct;
|
||||||
|
|
||||||
|
@ -23,7 +24,6 @@ import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
|
||||||
import eu.dnetlib.manager.wf.model.WorkflowGraph;
|
import eu.dnetlib.manager.wf.model.WorkflowGraph;
|
||||||
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
||||||
import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository;
|
import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository;
|
||||||
import eu.dnetlib.manager.wf.workflows.util.ProcessCallback;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
|
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
|
||||||
import eu.dnetlib.utils.Stoppable;
|
import eu.dnetlib.utils.Stoppable;
|
||||||
import eu.dnetlib.utils.StoppableDetails;
|
import eu.dnetlib.utils.StoppableDetails;
|
||||||
|
@ -63,7 +63,12 @@ public class WorkflowExecutor implements Stoppable {
|
||||||
}, 10, 10, TimeUnit.SECONDS);
|
}, 10, 10, TimeUnit.SECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String startRepoHiWorkflow(final String wfId, final String dsId, final String apiId, final ProcessCallback processCallback, final String parent)
|
public String startRepoHiWorkflow(final String wfId,
|
||||||
|
final String dsId,
|
||||||
|
final String apiId,
|
||||||
|
final String parent,
|
||||||
|
final Consumer<WorkflowProcess> onSuccess,
|
||||||
|
final Consumer<WorkflowProcess> onFail)
|
||||||
throws WorkflowManagerException {
|
throws WorkflowManagerException {
|
||||||
|
|
||||||
if (isPaused()) {
|
if (isPaused()) {
|
||||||
|
@ -91,13 +96,16 @@ public class WorkflowExecutor implements Stoppable {
|
||||||
instance.setSystemParams(new HashMap<>());
|
instance.setSystemParams(new HashMap<>());
|
||||||
instance.setUserParams(new HashMap<>());
|
instance.setUserParams(new HashMap<>());
|
||||||
|
|
||||||
return startWorkflowInstance(instance, processCallback, parent);
|
return startWorkflowInstance(instance, parent, onSuccess, onFail);
|
||||||
} catch (final DsmException e) {
|
} catch (final DsmException e) {
|
||||||
throw new WorkflowManagerException("Invalid datasource: " + dsId, e);
|
throw new WorkflowManagerException("Invalid datasource: " + dsId, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String startWorkflowInstance(final String wfInstanceId, final ProcessCallback processCallback, final String parent) throws Exception {
|
public String startWorkflowInstance(final String wfInstanceId,
|
||||||
|
final String parent,
|
||||||
|
final Consumer<WorkflowProcess> onSuccess,
|
||||||
|
final Consumer<WorkflowProcess> onFail) throws Exception {
|
||||||
|
|
||||||
if (isPaused()) {
|
if (isPaused()) {
|
||||||
log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown");
|
log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown");
|
||||||
|
@ -106,10 +114,13 @@ public class WorkflowExecutor implements Stoppable {
|
||||||
|
|
||||||
final WorkflowInstance instance =
|
final WorkflowInstance instance =
|
||||||
workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId));
|
workflowInstanceRepository.findById(wfInstanceId).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + wfInstanceId));
|
||||||
return startWorkflowInstance(instance, processCallback, parent);
|
return startWorkflowInstance(instance, parent, onSuccess, onFail);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String startWorkflowInstance(final WorkflowInstance wfInstance, final ProcessCallback processCallback, final String parent)
|
public String startWorkflowInstance(final WorkflowInstance wfInstance,
|
||||||
|
final String parent,
|
||||||
|
final Consumer<WorkflowProcess> onSuccess,
|
||||||
|
final Consumer<WorkflowProcess> onFail)
|
||||||
throws WorkflowManagerException {
|
throws WorkflowManagerException {
|
||||||
|
|
||||||
if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) {
|
if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) {
|
||||||
|
@ -134,7 +145,7 @@ public class WorkflowExecutor implements Stoppable {
|
||||||
.orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId()));
|
.orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId()));
|
||||||
|
|
||||||
final WorkflowProcess process =
|
final WorkflowProcess process =
|
||||||
processFactory.newProcess(wfMetadata, wfGraph, wfInstance, processCallback, parent);
|
processFactory.newProcess(wfMetadata, wfGraph, wfInstance, parent, onSuccess, onFail);
|
||||||
|
|
||||||
return processRegistry.registerProcess(process, wfInstance);
|
return processRegistry.registerProcess(process, wfInstance);
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,16 +2,19 @@ package eu.dnetlib.manager.wf.workflows.procs;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.CopyOnWriteArrayList;
|
import java.util.concurrent.CopyOnWriteArrayList;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import org.apache.commons.lang3.math.NumberUtils;
|
import org.apache.commons.lang3.math.NumberUtils;
|
||||||
|
|
||||||
import eu.dnetlib.is.model.resource.SimpleResource;
|
import eu.dnetlib.is.model.resource.SimpleResource;
|
||||||
|
import eu.dnetlib.manager.history.model.WfProcessExecution;
|
||||||
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
import eu.dnetlib.manager.wf.model.WorkflowInstance;
|
||||||
import eu.dnetlib.manager.wf.workflows.graph.Graph;
|
import eu.dnetlib.manager.wf.workflows.graph.Graph;
|
||||||
import eu.dnetlib.manager.wf.workflows.util.ProcessCallback;
|
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Created by michele on 19/11/15.
|
* Created by michele on 19/11/15.
|
||||||
|
@ -36,7 +39,8 @@ public class WorkflowProcess implements Comparable<WorkflowProcess> {
|
||||||
private final SimpleResource wfMetadata;
|
private final SimpleResource wfMetadata;
|
||||||
private final WorkflowInstance wfInstance;
|
private final WorkflowInstance wfInstance;
|
||||||
private final Graph graph;
|
private final Graph graph;
|
||||||
private final ProcessCallback callback;
|
private final Consumer<WorkflowProcess> onSuccess;
|
||||||
|
private final Consumer<WorkflowProcess> onFail;
|
||||||
private final Env env;
|
private final Env env;
|
||||||
private final List<Token> tokens = new CopyOnWriteArrayList<>();
|
private final List<Token> tokens = new CopyOnWriteArrayList<>();
|
||||||
private LocalDateTime lastActivityDate;
|
private LocalDateTime lastActivityDate;
|
||||||
|
@ -56,13 +60,15 @@ public class WorkflowProcess implements Comparable<WorkflowProcess> {
|
||||||
final WorkflowInstance wfInstance,
|
final WorkflowInstance wfInstance,
|
||||||
final Graph graph,
|
final Graph graph,
|
||||||
final Map<String, String> globalParams,
|
final Map<String, String> globalParams,
|
||||||
final ProcessCallback callback,
|
final String parentProfileId,
|
||||||
final String parentProfileId) {
|
final Consumer<WorkflowProcess> onSuccess,
|
||||||
|
final Consumer<WorkflowProcess> onFail) {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
this.wfMetadata = wfMetadata;
|
this.wfMetadata = wfMetadata;
|
||||||
this.wfInstance = wfInstance;
|
this.wfInstance = wfInstance;
|
||||||
this.graph = graph;
|
this.graph = graph;
|
||||||
this.callback = callback;
|
this.onSuccess = onSuccess;
|
||||||
|
this.onFail = onFail;
|
||||||
this.status = Status.CREATED;
|
this.status = Status.CREATED;
|
||||||
this.env = new Env();
|
this.env = new Env();
|
||||||
this.globalParams = globalParams;
|
this.globalParams = globalParams;
|
||||||
|
@ -134,10 +140,6 @@ public class WorkflowProcess implements Comparable<WorkflowProcess> {
|
||||||
setStatus(Status.KILLED);
|
setStatus(Status.KILLED);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ProcessCallback getCallback() {
|
|
||||||
return callback;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isTerminated() {
|
public boolean isTerminated() {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case SUCCESS:
|
case SUCCESS:
|
||||||
|
@ -211,4 +213,56 @@ public class WorkflowProcess implements Comparable<WorkflowProcess> {
|
||||||
return parentProfileId;
|
return parentProfileId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void complete(final Token token) {
|
||||||
|
final LocalDateTime now = token.getEndDate();
|
||||||
|
setLastActivityDate(now);
|
||||||
|
setEndDate(now);
|
||||||
|
setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS);
|
||||||
|
|
||||||
|
if (token.isFailed()) {
|
||||||
|
setStatus(Status.FAILURE);
|
||||||
|
setError(token.getError());
|
||||||
|
setErrorStacktrace(token.getErrorStackTrace());
|
||||||
|
setLastActivityDate(LocalDateTime.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (token.isFailed() && onFail != null) {
|
||||||
|
onFail.accept(this);
|
||||||
|
}
|
||||||
|
if (!token.isFailed() && onSuccess != null) {
|
||||||
|
onSuccess.accept(this);;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public WfProcessExecution asLog() {
|
||||||
|
final Map<String, String> details = new LinkedHashMap<>();
|
||||||
|
details.putAll(getOutputParams());
|
||||||
|
details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + getPriority());
|
||||||
|
details.put(WorkflowsConstants.LOG_WF_ID, getWfId());
|
||||||
|
details.put(WorkflowsConstants.LOG_WF_ID, getWfInstanceId());
|
||||||
|
|
||||||
|
if (getError() != null) {
|
||||||
|
details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, getError());
|
||||||
|
details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, getErrorStacktrace());
|
||||||
|
}
|
||||||
|
|
||||||
|
final WfProcessExecution pe = new WfProcessExecution();
|
||||||
|
pe.setProcessId(getId());
|
||||||
|
pe.setName(getName());
|
||||||
|
pe.setFamily(getFamily());
|
||||||
|
|
||||||
|
pe.setDsId(getDsId());
|
||||||
|
pe.setDsName(getDsName());
|
||||||
|
pe.setDsApi(getDsInterface());
|
||||||
|
|
||||||
|
pe.setStartDate(getStartDate());
|
||||||
|
pe.setEndDate(getEndDate());
|
||||||
|
pe.setStatus(getStatus().toString());
|
||||||
|
|
||||||
|
pe.setDetails(details);
|
||||||
|
|
||||||
|
return pe;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,13 +0,0 @@
|
||||||
package eu.dnetlib.manager.wf.workflows.util;
|
|
||||||
|
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.Token;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Created by michele on 26/11/15.
|
|
||||||
*/
|
|
||||||
public interface NodeTokenCallback {
|
|
||||||
|
|
||||||
void onSuccess(Token token);
|
|
||||||
|
|
||||||
void onFail(Token token);
|
|
||||||
}
|
|
|
@ -1,12 +0,0 @@
|
||||||
package eu.dnetlib.manager.wf.workflows.util;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Created by michele on 18/11/15.
|
|
||||||
*/
|
|
||||||
public interface ProcessCallback {
|
|
||||||
|
|
||||||
void onSuccess();
|
|
||||||
|
|
||||||
void onFail();
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,6 +0,0 @@
|
||||||
package eu.dnetlib.manager.wf.workflows.util;
|
|
||||||
|
|
||||||
public interface ProgressProvider {
|
|
||||||
|
|
||||||
String getProgressDescription();
|
|
||||||
}
|
|
|
@ -1,37 +0,0 @@
|
||||||
package eu.dnetlib.manager.wf.workflows.util;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
|
|
||||||
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
|
|
||||||
|
|
||||||
public class SubWorkflowProgressProvider implements ProgressProvider {
|
|
||||||
|
|
||||||
private final String procId;
|
|
||||||
private final ProcessRegistry processRegistry;
|
|
||||||
|
|
||||||
public SubWorkflowProgressProvider(final String procId, final ProcessRegistry processRegistry) {
|
|
||||||
super();
|
|
||||||
this.procId = procId;
|
|
||||||
this.processRegistry = processRegistry;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getProgressDescription() {
|
|
||||||
final WorkflowProcess proc = this.processRegistry.findProcess(this.procId);
|
|
||||||
|
|
||||||
if (proc == null) { return "-"; }
|
|
||||||
|
|
||||||
final List<String> list = proc.getTokens()
|
|
||||||
.stream()
|
|
||||||
.filter(t -> t.isActive())
|
|
||||||
.map(t -> t.getProgressProvider() != null ? String.format("%s (%s)", t.getNodeName(), t.getProgressProvider().getProgressDescription())
|
|
||||||
: t.getNodeName())
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
if (!list.isEmpty()) { return list.stream().collect(Collectors.joining(", ")); }
|
|
||||||
|
|
||||||
return "-";
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
Loading…
Reference in New Issue