diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryController.java new file mode 100644 index 00000000..f1f25f57 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryController.java @@ -0,0 +1,14 @@ +package eu.dnetlib.is.wfs; + +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.GetMapping; + +@Controller +public class WfHistoryController { + + @GetMapping("/wf_history") + public void wfHistory(final ModelMap map) { + + } +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryImporter.java new file mode 100644 index 00000000..d25b9b79 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryImporter.java @@ -0,0 +1,81 @@ +package eu.dnetlib.is.wfs; + +import java.io.File; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.is.wf.model.WfProcessExecution; +import eu.dnetlib.is.wf.repository.WfProcessExecutionRepository; + +@Component +public class WfHistoryImporter { + + private static final Log log = LogFactory.getLog(WfHistoryImporter.class); + + @Autowired + private WfProcessExecutionRepository wfProcessExecutionRepository; + + public void load(final String path) throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + + final JsonNode rootNode = mapper.readTree(new File(path)); + + rootNode.forEach(wf -> saveWf(wf)); + } + + private void saveWf(final JsonNode node) { + + final WfProcessExecution wf = new WfProcessExecution(); + wf.setProcessId(node.get("system:processId").asText()); + wf.setName(node.get("system:wfName").asText()); + wf.setFamily(node.get("system:profileFamily").asText()); + + if (node.has("dataprovider:id")) { + wf.setDsId(node.get("dataprovider:id").asText()); + } + if (node.has("dataprovider:name")) { + wf.setDsName(node.get("dataprovider:name").asText()); + } + if (node.has("dataprovider:interface")) { + wf.setDsApi(node.get("dataprovider:interface").asText()); + } + + wf.setStartDate(new Date(NumberUtils.toLong(node.get("system:startDate").asText()))); + wf.setEndDate(new Date(NumberUtils.toLong(node.get("system:endDate").asText()))); + + if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) { + wf.setStatus("success"); + } else { + wf.setStatus("failure"); + } + + final Map details = new LinkedHashMap<>(); + final Iterator> fields = node.fields(); + while (fields.hasNext()) { + final Entry f = fields.next(); + if (f.getValue().isValueNode()) { + details.put(f.getKey(), f.getValue().asText()); + } + } + + wf.setDetails(details); + + wfProcessExecutionRepository.save(wf); + + log.info("Wf saved with id: " + wf.getProcessId()); + + } +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryRestController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryRestController.java new file mode 100644 index 00000000..d863bee6 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/wfs/WfHistoryRestController.java @@ -0,0 +1,56 @@ +package eu.dnetlib.is.wfs; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.is.wf.model.WfProcessExecution; +import eu.dnetlib.is.wf.repository.WfProcessExecutionRepository; + +@RestController +@RequestMapping("/api/wfs") +public class WfHistoryRestController { + + @Autowired + private WfProcessExecutionRepository wfProcessExecutionRepository; + + @Autowired + private WfHistoryImporter wfHistoryImporter; + + @GetMapping("/") + public List history(@RequestParam(required = false) final Date from, @RequestParam(required = false) final Date to) { + if (from == null && to == null) { + return wfProcessExecutionRepository.findAll(PageRequest.of(0, 100, Sort.by("endDate").descending())).toList(); + } else if (from == null) { + return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(0), to); + } else if (to == null) { + return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(from, new Date()); + } else { + return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(from, to); + } + } + + @GetMapping("/{processId}") + public WfProcessExecution getProcess(@PathVariable final String processId) { + return wfProcessExecutionRepository.findById(processId).get(); + } + + @GetMapping(value = "/load") + public List loadFromOldProfile(@RequestParam final String path) throws Exception { + // mongoexport -d dnet_logs -c wf_logs --jsonArray -o /tmp/mongodump.json + + wfHistoryImporter.load(path); + + return Arrays.asList("Done."); + } + +} diff --git a/apps/dnet-is-application/src/main/resources/static/js/wf_history.js b/apps/dnet-is-application/src/main/resources/static/js/wf_history.js new file mode 100644 index 00000000..cc177e6c --- /dev/null +++ b/apps/dnet-is-application/src/main/resources/static/js/wf_history.js @@ -0,0 +1,22 @@ +var app = angular.module('wfHistoryApp', []); + +app.controller('wfHistoryController', function($scope, $http) { + $scope.workflows = []; + $scope.currentWf = {}; + + + $scope.reload = function() { + $http.get('./api/wfs/?' + $.now()).then(function successCallback(res) { + $scope.workflows = res.data; + }, function errorCallback(res) { + alert('ERROR: ' + res.data.message); + }); + }; + + $scope.setCurrentWf = function(wf) { + $scope.currentWf = angular.copy(wf); + } + + + $scope.reload(); +}); diff --git a/apps/dnet-is-application/src/main/resources/templates/fragments/mainParts.html b/apps/dnet-is-application/src/main/resources/templates/fragments/mainParts.html index fa8a8039..b94c1d9b 100644 --- a/apps/dnet-is-application/src/main/resources/templates/fragments/mainParts.html +++ b/apps/dnet-is-application/src/main/resources/templates/fragments/mainParts.html @@ -36,6 +36,12 @@ Transformation rules +
  • + Logs +
    + Workflow history +
    +
    • diff --git a/apps/dnet-is-application/src/main/resources/templates/wf_history.html b/apps/dnet-is-application/src/main/resources/templates/wf_history.html new file mode 100644 index 00000000..4c108b0f --- /dev/null +++ b/apps/dnet-is-application/src/main/resources/templates/wf_history.html @@ -0,0 +1,75 @@ + + + + + + + + + +
    +
    +
    + +

    + +

    +

    + Total workflows : {{(workflows | filter:wfFilter).length}} +

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Process IdWorkflow NameWorkflow FamilyDatasourceStatusStart DateEnd Date
    no workflows
    {{wf.processId}}{{wf.name}}{{wf.family}}{{wf.dsName}}{{wf.status}}{{wf.startDate}}{{wf.endDate}}
    +
    +
    +
    + + + + + + + + + + + \ No newline at end of file diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/model/WfProcessExecution.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/model/WfProcessExecution.java new file mode 100644 index 00000000..694e995e --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/model/WfProcessExecution.java @@ -0,0 +1,145 @@ +package eu.dnetlib.is.wf.model; + +import java.io.Serializable; +import java.util.Date; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; +import com.vladmihalcea.hibernate.type.json.JsonStringType; + +@Entity +@Table(name = "wf_history") +@TypeDefs({ + @TypeDef(name = "json", typeClass = JsonStringType.class), + @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) +}) +public class WfProcessExecution implements Serializable { + + private static final long serialVersionUID = -326994850248506828L; + + @Id + @Column(name = "process_id") + private String processId; + + @Column(name = "name") + private String name; + + @Column(name = "family") + private String family; + + @Column(name = "status") + private String status; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "start_date") + private Date startDate; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "end_date") + private Date endDate; + + @Column(name = "ds_id") + private String dsId; + + @Column(name = "ds_name") + private String dsName; + + @Column(name = "ds_api") + private String dsApi; + + @Type(type = "jsonb") + @Column(name = "details") + private Map details; + + public String getProcessId() { + return processId; + } + + public void setProcessId(final String processId) { + this.processId = processId; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getFamily() { + return family; + } + + public void setFamily(final String family) { + this.family = family; + } + + public String getStatus() { + return status; + } + + public void setStatus(final String status) { + this.status = status; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(final Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(final Date endDate) { + this.endDate = endDate; + } + + public String getDsId() { + return dsId; + } + + public void setDsId(final String dsId) { + this.dsId = dsId; + } + + public String getDsName() { + return dsName; + } + + public void setDsName(final String dsName) { + this.dsName = dsName; + } + + public String getDsApi() { + return dsApi; + } + + public void setDsApi(final String dsApi) { + this.dsApi = dsApi; + } + + public Map getDetails() { + return details; + } + + public void setDetails(final Map details) { + this.details = details; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/repository/WfProcessExecutionRepository.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/repository/WfProcessExecutionRepository.java new file mode 100644 index 00000000..a649c272 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/wf/repository/WfProcessExecutionRepository.java @@ -0,0 +1,13 @@ +package eu.dnetlib.is.wf.repository; + +import java.util.Date; +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.is.wf.model.WfProcessExecution; + +public interface WfProcessExecutionRepository extends JpaRepository { + + List findByEndDateBetweenOrderByEndDateDesc(Date start, Date end); +} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index 656e785d..fb92a324 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -63,3 +63,17 @@ CREATE INDEX ON context_cat_concepts_lvl_0 (parent); CREATE INDEX ON context_cat_concepts_lvl_1 (parent); CREATE INDEX ON context_cat_concepts_lvl_2 (parent); +-- WF History + +CREATE TABLE wf_history ( + process_id text PRIMARY KEY, + name text NOT NULL, + family text NOT NULL, + status text NOT NULL, + start_date timestamp NOT NULL, + end_date timestamp NOT NULL, + ds_id text, + ds_name text, + ds_api text, + details jsonb +); \ No newline at end of file