diff --git a/apps/dnet-is-application/pom.xml b/apps/dnet-is-application/pom.xml index f563dabe..92bde558 100644 --- a/apps/dnet-is-application/pom.xml +++ b/apps/dnet-is-application/pom.xml @@ -20,12 +20,18 @@ ${project.version} + + eu.dnetlib.dhp + dnet-wf-service + ${project.version} + + eu.dnetlib.dhp dnet-data-services ${project.version} - + org.springframework.boot diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java index 0a5cb752..d3b01f91 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java @@ -20,7 +20,11 @@ import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration @EnableTransactionManagement @EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = { - "eu.dnetlib.is", "eu.dnetlib.data.mdstore", "eu.dnetlib.msro" + "eu.dnetlib.is", + "eu.dnetlib.common", + "eu.dnetlib.manager.history", + "eu.dnetlib.manager.wf", + "eu.dnetlib.data.mdstore" }) public class MainDBConfig { @@ -38,7 +42,7 @@ public class MainDBConfig { @Qualifier("mainDataSource") final DataSource ds) { return builder .dataSource(ds) - .packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.data.mdstore.model") + .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model") .persistenceUnit("is") .build(); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java index 55a4590e..d5ff171b 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinAjaxController.java @@ -21,7 +21,6 @@ public class ZeppelinAjaxController extends AbstractDnetController { @GetMapping("/templates") public List getTemplates() throws MDStoreManagerException { try { - // if (zeppelinClient.get) return zeppelinClient.listTemplates(); } catch (final Throwable e) { throw new MDStoreManagerException("Zeppelin is unreachable", e); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java index 330d8e95..a9d49ea1 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmAjaxController.java @@ -40,9 +40,9 @@ public class DsmAjaxController extends AbstractDnetController { private ProtocolService protocolService; @GetMapping("/browsableFields") - public List browsableFields() { + public List> browsableFields() { return Arrays.stream(DsmBrowsableFields.values()) - .map(f -> new KeyValue(f.name(), f.desc)) + .map(f -> new KeyValue<>(f.name(), f.desc)) .collect(Collectors.toList()); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java new file mode 100644 index 00000000..425e9ce5 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/email/EmailTemplateController.java @@ -0,0 +1,41 @@ +package eu.dnetlib.is.email; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.common.model.EmailTemplate; +import eu.dnetlib.notifications.mail.EmailService; + +@RestController +@RequestMapping("/ajax/templates/email") +public class EmailTemplateController extends AbstractDnetController { + + @Autowired + protected EmailService emailService; + + @GetMapping("/") + public List listEmailTemplates() { + return emailService.listEmailTemplates(); + } + + @PostMapping("/") + public List saveEmailTemplate(@RequestBody final EmailTemplate email) { + emailService.saveEmailTemplate(email); + return emailService.listEmailTemplates(); + } + + @DeleteMapping("/{id}") + public List deleteEmailTemplate(@PathVariable final String id) { + emailService.deleteEmailTemplate(id); + return emailService.listEmailTemplates(); + } +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java index 58a395d8..e37519b2 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java @@ -1,6 +1,6 @@ package eu.dnetlib.is.importer; -import java.util.Date; +import java.time.LocalDateTime; import javax.transaction.Transactional; @@ -47,7 +47,7 @@ public class OldProfilesImporter { final Document doc = DocumentHelper.parseText(xml); final String id = StringUtils.substringBefore(doc.valueOf("//RESOURCE_IDENTIFIER/@value"), "_"); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); final SimpleResource res = new SimpleResource(); res.setId(id); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java index 95d1d6c2..1a6c6b27 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/WfHistoryImporter.java @@ -1,11 +1,13 @@ package eu.dnetlib.is.importer; import java.io.File; -import java.util.Date; +import java.time.Instant; +import java.time.LocalDateTime; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.TimeZone; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -17,8 +19,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository; -import eu.dnetlib.msro.model.history.WfProcessExecution; +import eu.dnetlib.manager.history.model.WfProcessExecution; +import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository; @Service public class WfHistoryImporter { @@ -53,8 +55,10 @@ public class WfHistoryImporter { wf.setDsApi(node.get("dataprovider:interface").asText()); } - wf.setStartDate(new Date(NumberUtils.toLong(node.get("system:startDate").asText()))); - wf.setEndDate(new Date(NumberUtils.toLong(node.get("system:endDate").asText()))); + wf.setStartDate(LocalDateTime + .ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:startDate").asText())), TimeZone.getDefault().toZoneId())); + + wf.setEndDate(LocalDateTime.ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:endDate").asText())), TimeZone.getDefault().toZoneId())); if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) { wf.setStatus("success"); diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java index 8b98de3c..6b0f2b8b 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/InfoAjaxController.java @@ -56,30 +56,30 @@ public class InfoAjaxController extends AbstractDnetController { return res; } - private InfoSection jvm() { - final InfoSection jvm = new InfoSection<>("JVM"); - jvm.getData().add(new KeyValue("JVM Name", mxbean.getVmName())); - jvm.getData().add(new KeyValue("JVM Vendor", mxbean.getVmVendor())); - jvm.getData().add(new KeyValue("JVM Version", mxbean.getVmVersion())); - jvm.getData().add(new KeyValue("JVM Spec Name", mxbean.getSpecName())); - jvm.getData().add(new KeyValue("JVM Spec Vendor", mxbean.getSpecVendor())); - jvm.getData().add(new KeyValue("JVM Spec Version", mxbean.getSpecVersion())); - jvm.getData().add(new KeyValue("Running JVM Name", mxbean.getName())); - jvm.getData().add(new KeyValue("Management Spec Version", mxbean.getManagementSpecVersion())); + private InfoSection> jvm() { + final InfoSection> jvm = new InfoSection<>("JVM"); + jvm.getData().add(new KeyValue<>("JVM Name", mxbean.getVmName())); + jvm.getData().add(new KeyValue<>("JVM Vendor", mxbean.getVmVendor())); + jvm.getData().add(new KeyValue<>("JVM Version", mxbean.getVmVersion())); + jvm.getData().add(new KeyValue<>("JVM Spec Name", mxbean.getSpecName())); + jvm.getData().add(new KeyValue<>("JVM Spec Vendor", mxbean.getSpecVendor())); + jvm.getData().add(new KeyValue<>("JVM Spec Version", mxbean.getSpecVersion())); + jvm.getData().add(new KeyValue<>("Running JVM Name", mxbean.getName())); + jvm.getData().add(new KeyValue<>("Management Spec Version", mxbean.getManagementSpecVersion())); return jvm; } - private InfoSection args() { - final InfoSection libs = new InfoSection<>("Arguments"); - libs.getData().add(new KeyValue("Input arguments", StringUtils.join(mxbean.getInputArguments(), " "))); + private InfoSection> args() { + final InfoSection> libs = new InfoSection<>("Arguments"); + libs.getData().add(new KeyValue<>("Input arguments", StringUtils.join(mxbean.getInputArguments(), " "))); return libs; } - private List> props() { - final List> res = new ArrayList<>(); + private List>> props() { + final List>> res = new ArrayList<>(); configurableEnvironment.getPropertySources().forEach(ps -> { - final InfoSection section = new InfoSection<>("Properties: " + ps.getName()); + final InfoSection> section = new InfoSection<>("Properties: " + ps.getName()); addAllProperties(section, ps); res.add(section); }); @@ -87,13 +87,13 @@ public class InfoAjaxController extends AbstractDnetController { return res; } - private void addAllProperties(final InfoSection res, final PropertySource ps) { + private void addAllProperties(final InfoSection> res, final PropertySource ps) { if (ps instanceof CompositePropertySource) { final CompositePropertySource cps = (CompositePropertySource) ps; cps.getPropertySources().forEach(x -> addAllProperties(res, x)); } else if (ps instanceof EnumerablePropertySource) { final EnumerablePropertySource eps = (EnumerablePropertySource) ps; - Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue(k, eps.getProperty(k)))); + Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue<>(k, eps.getProperty(k)))); } else {} } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java index 082e63fb..e7abe9f9 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/info/KeyValue.java @@ -1,11 +1,11 @@ package eu.dnetlib.is.info; -public class KeyValue { +public class KeyValue { private final String k; - private final Object v; + private final T v; - public KeyValue(final String k, final Object v) { + public KeyValue(final String k, final T v) { this.k = k; this.v = v; } @@ -14,7 +14,7 @@ public class KeyValue { return k; } - public Object getV() { + public T getV() { return v; } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java index e154e2f8..a9ec2707 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceAjaxController.java @@ -25,11 +25,12 @@ public class ResourceAjaxController extends AbstractResourceController { @PostMapping("/") public SimpleResource newResource(@RequestParam final String name, @RequestParam final String type, + @RequestParam(required = false, defaultValue = "") final String subtype, @RequestParam(required = false, defaultValue = "") final String description, @RequestParam final String content) throws InformationServiceException { - return service.saveNewResource(name, type, description, content); + return service.saveNewResource(name, type, subtype, description, content); } @DeleteMapping("/{resId}") diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java new file mode 100644 index 00000000..0993426b --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfHistoryAjaxController.java @@ -0,0 +1,36 @@ +package eu.dnetlib.manager.wf; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.history.model.WfProcessExecution; + +@RestController +@RequestMapping("/ajax/wf_history") +public class WfHistoryAjaxController extends AbstractDnetController { + + @Autowired + private WorkflowLogger logger; + + @GetMapping("/") + public List history( + @RequestParam(required = true) final int total, + @RequestParam(required = false) final Long from, + @RequestParam(required = false) final Long to) { + return logger.history(total, from, to); + } + + @GetMapping("/{processId}") + public WfProcessExecution getProcessExecution(@PathVariable final String processId) { + return logger.getProcessExecution(processId); + } + +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java new file mode 100644 index 00000000..23e22ad5 --- /dev/null +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/manager/wf/WfInstancesController.java @@ -0,0 +1,41 @@ +package eu.dnetlib.manager.wf; + +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.common.controller.AbstractDnetController; +import eu.dnetlib.is.info.KeyValue; +import eu.dnetlib.manager.wf.model.WorkflowInstance; + +@RestController +@RequestMapping("/ajax/wf_instances") +public class WfInstancesController extends AbstractDnetController { + + private WorkflowManagerService wfManagerService; + + @GetMapping("/instance/{id}") + public WorkflowInstance getWfInstance(@PathVariable final String id) throws Exception { + return wfManagerService.findWorkflowInstance(id); + } + + @GetMapping("/search") + public List> listWfInstances(@RequestParam final String section) throws Exception { + return wfManagerService.streamWfInstancesBySection(section) + .map(x -> new KeyValue<>(x.getId(), x.getName())) + .collect(Collectors.toList()); + } + + @GetMapping("/sections") + public List> listWfFamilies() throws Exception { + return wfManagerService.streamSections() + .map(x -> new KeyValue<>(x.getValue(), x.getCount())) + .collect(Collectors.toList()); + } + +} diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java deleted file mode 100644 index a6ccc190..00000000 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/msro/history/WfHistoryAjaxController.java +++ /dev/null @@ -1,49 +0,0 @@ -package eu.dnetlib.msro.history; - -import java.util.Date; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository; -import eu.dnetlib.msro.model.history.WfProcessExecution; - -@RestController -@RequestMapping("/ajax/wfs") -public class WfHistoryAjaxController { - - @Autowired - private WfProcessExecutionRepository wfProcessExecutionRepository; - - @Deprecated - public static final int MAX_NUMBER_OF_RECENT_WFS = 100; - - @GetMapping("/") - public List history( - @RequestParam(required = true) final int total, - @RequestParam(required = false) final Long from, - @RequestParam(required = false) final Long to) { - if (from == null && to == null) { - return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList(); - } else if (from == null) { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(0), new Date(to)); - } else if (to == null) { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date()); - } else { - return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date(to)); - } - } - - @GetMapping("/{processId}") - public WfProcessExecution getProcess(@PathVariable final String processId) { - return wfProcessExecutionRepository.findById(processId).get(); - } - -} diff --git a/apps/dnet-is-application/src/main/resources/application.properties b/apps/dnet-is-application/src/main/resources/application.properties index f56fd65d..a2a29b36 100644 --- a/apps/dnet-is-application/src/main/resources/application.properties +++ b/apps/dnet-is-application/src/main/resources/application.properties @@ -3,6 +3,8 @@ server.port=8280 server.public_url = server.public_desc = API Base URL +dnet.configuration.infrastructure = LOCAL DEV + spring.profiles.active=dev maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-pom.xml @@ -57,5 +59,12 @@ dhp.mdstore-manager.hadoop.zeppelin.login = dhp.mdstore-manager.hadoop.zeppelin.password = dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager - +# Email Configuration +dnet.configuration.mail.sender.email = +dnet.configuration.mail.sender.name = +dnet.configuration.mail.cc = +dnet.configuration.mail.smtp.host = +dnet.configuration.mail.smtp.port = 25 +dnet.configuration.mail.smtp.user = +dnet.configuration.mail.smtp.password = diff --git a/frontends/dnet-is-application/src/app/app-routing.module.ts b/frontends/dnet-is-application/src/app/app-routing.module.ts index ae8898c5..b2a848b5 100644 --- a/frontends/dnet-is-application/src/app/app-routing.module.ts +++ b/frontends/dnet-is-application/src/app/app-routing.module.ts @@ -9,6 +9,8 @@ import { ContextViewerComponent, ContextsComponent } from './contexts/contexts.c import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/dsm.component'; import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; +import { EmailsComponent } from './emails/emails.component'; +import { WfInstancesComponent } from './wf-instances/wf-instances.component'; const routes: Routes = [ { path: "", redirectTo: 'info', pathMatch: 'full' }, @@ -17,6 +19,8 @@ const routes: Routes = [ { path: "adv_resources/context", component: ContextsComponent }, { path: "adv_resources/vocabulary", component: VocabulariesComponent }, { path: "adv_resources/protocol", component: ProtocolsComponent }, + { path: "adv_resources/email", component: EmailsComponent }, + { path: "wfs", component: WfInstancesComponent }, { path: "wf_history", component: WfHistoryComponent }, { path: "ctx_viewer", component: ContextViewerComponent }, { path: "voc_editor", component: VocabularyEditorComponent }, diff --git a/frontends/dnet-is-application/src/app/app.module.ts b/frontends/dnet-is-application/src/app/app.module.ts index 5ae2c16d..59ad6a0c 100644 --- a/frontends/dnet-is-application/src/app/app.module.ts +++ b/frontends/dnet-is-application/src/app/app.module.ts @@ -36,6 +36,9 @@ import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; import { SpinnerHttpInterceptor } from './common/spinner.service'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; +import { EmailDialog, EmailsComponent } from './emails/emails.component'; +import { WfInstancesComponent } from './wf-instances/wf-instances.component'; +import { MatTabsModule } from '@angular/material/tabs'; @NgModule({ declarations: [ @@ -66,7 +69,10 @@ import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.componen MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog, - CleanerTesterComponent + CleanerTesterComponent, + EmailsComponent, + EmailDialog, + WfInstancesComponent ], imports: [ BrowserModule, @@ -95,7 +101,8 @@ import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.componen ReactiveFormsModule, MatSnackBarModule, MatPaginatorModule, - MatProgressSpinnerModule + MatProgressSpinnerModule, + MatTabsModule ], providers: [{ provide: HTTP_INTERCEPTORS, diff --git a/frontends/dnet-is-application/src/app/common/is.model.ts b/frontends/dnet-is-application/src/app/common/is.model.ts index b6de0432..59faf1e4 100644 --- a/frontends/dnet-is-application/src/app/common/is.model.ts +++ b/frontends/dnet-is-application/src/app/common/is.model.ts @@ -54,6 +54,7 @@ export interface SimpleResource { id: string, name: string, type: string, + subtype?: string, description?: string, creationDate?: string, modificationDate?: string @@ -198,3 +199,29 @@ export interface MDStoreRecord { dateOfTransformation: string, provenance: any } + +export interface EmailTemplate { + id: string, + description: string, + subject: string, + message: string +} + + +export interface WfInstance { + id: string, + details: Map, + priority: number, + dsId?: string, + dsName?: string, + apiId?: string, + enabled: boolean, + configured: boolean, + schedulingEnabled: boolean, + cronExpression?: string, + cronMinInterval?: number, + workflow: string, + destroyWf?: string, + systemParams: Map, + userParams: Map +} diff --git a/frontends/dnet-is-application/src/app/common/is.service.ts b/frontends/dnet-is-application/src/app/common/is.service.ts index ecef739e..80fa5a04 100644 --- a/frontends/dnet-is-application/src/app/common/is.service.ts +++ b/frontends/dnet-is-application/src/app/common/is.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; -import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model'; +import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord, EmailTemplate } from './is.model'; import { FormGroup } from '@angular/forms'; import { MatSnackBar } from '@angular/material/snack-bar'; @@ -72,11 +72,12 @@ export class ISService { }); } - addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { + addSimpleResource(name: string, type: string, subtype: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') let body = new HttpParams() .set('name', name) .set('type', type) + .set('subtype', subtype) .set('description', description) .set('content', content); this.client.post('/ajax/resources/', body, { headers: headers }).subscribe({ @@ -98,7 +99,7 @@ export class ISService { if (from && from > 0) { params = params.append('from', from); } if (to && to > 0) { params = params.append('to', to); } - this.client.get('/ajax/wfs/', { params: params }).subscribe({ + this.client.get('/ajax/wf_history/', { params: params }).subscribe({ next: data => onSuccess(data), error: error => this.showError(error) }); @@ -328,6 +329,47 @@ export class ISService { }); } + loadEmailTemplates(onSuccess: Function): void { + this.client.get('./ajax/templates/email/').subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + saveEmailTemplate(email: EmailTemplate, onSuccess: Function, relatedForm?: FormGroup): void { + this.client.post('./ajax/templates/email/', email).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error, relatedForm) + }); + } + + deleteEmailTemplate(id: string, onSuccess: Function): void { + this.client.delete('./ajax/templates/email/' + encodeURIComponent(id)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + loadWfIntancesSections(onSuccess: Function): void { + this.client.get('./ajax/wf_instances/sections').subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + loadWfIntances(section: string, onSuccess: Function): void { + this.client.get('./ajax/wf_instances/search?section=' + encodeURIComponent(section)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } + + loadWfIntance(id: string, onSuccess: Function): void { + this.client.get('./ajax/wf_instances/instance/' + encodeURIComponent(id)).subscribe({ + next: data => onSuccess(data), + error: error => this.showError(error) + }); + } private showError(error: any, form?: FormGroup) { diff --git a/frontends/dnet-is-application/src/app/emails/email-dialog.html b/frontends/dnet-is-application/src/app/emails/email-dialog.html new file mode 100644 index 00000000..4ad7e7c0 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/email-dialog.html @@ -0,0 +1,41 @@ +
+ +

Edit Email Template

+

New Email Template

+ +
+ + + ID + + + + + Description + + This field is required + + + + Email: subject + + This field is required + + + + Email: message + + This field is required + + +
+ +
+ + + + {{ emailForm.errors?.['serverError'] }} + +
+ +
diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.css b/frontends/dnet-is-application/src/app/emails/emails.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.html b/frontends/dnet-is-application/src/app/emails/emails.component.html new file mode 100644 index 00000000..f3b72955 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/emails.component.html @@ -0,0 +1,42 @@ +

Email Templates

+ + + + + Filter + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Id + + {{element.id}} + Description {{element.description}} + +
No data matching the filter "{{input.value}}"
diff --git a/frontends/dnet-is-application/src/app/emails/emails.component.ts b/frontends/dnet-is-application/src/app/emails/emails.component.ts new file mode 100644 index 00000000..6a47eae7 --- /dev/null +++ b/frontends/dnet-is-application/src/app/emails/emails.component.ts @@ -0,0 +1,98 @@ +import { AfterViewInit, Component, Inject, OnInit, ViewChild } from '@angular/core'; +import { FormControl, FormGroup, Validators } from '@angular/forms'; +import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MatSort } from '@angular/material/sort'; +import { MatTableDataSource } from '@angular/material/table'; +import { ActivatedRoute } from '@angular/router'; +import { EmailTemplate } from '../common/is.model'; +import { ISService } from '../common/is.service'; + +@Component({ + selector: 'app-emails', + templateUrl: './emails.component.html', + styleUrls: ['./emails.component.css'] +}) +export class EmailsComponent implements OnInit, AfterViewInit { + emailsDatasource: MatTableDataSource = new MatTableDataSource([]); + + colums: string[] = ['id', 'description', 'buttons']; + + @ViewChild(MatSort) sort: MatSort | undefined + + searchText: string = ''; + + constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) { } + + ngOnInit() { this.reload() } + ngAfterViewInit() { if (this.sort) this.emailsDatasource.sort = this.sort; } + reload() { this.service.loadEmailTemplates((data: EmailTemplate[]) => this.emailsDatasource.data = data); } + + applyFilter(event: Event) { + const filterValue = (event.target as HTMLInputElement).value.trim().toLowerCase(); + this.emailsDatasource.filter = filterValue; + } + + openAddEmailTemplateDialog(): void { + const dialogRef = this.dialog.open(EmailDialog, { + data: { + id: '', + description: '', + subject: '', + message: '' + }, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + openEditEmailTemplateDialog(email: EmailTemplate): void { + const dialogRef = this.dialog.open(EmailDialog, { + data: email, + width: '80%' + }); + + dialogRef.afterClosed().subscribe(result => { + if (result) this.reload(); + }); + } + + deleteEmailTemplate(email: EmailTemplate) { + if (confirm('Are you sure?')) { + this.service.deleteEmailTemplate(email.id, (data: void) => this.reload()); + } + } +} + +@Component({ + selector: 'email-dialog', + templateUrl: './email-dialog.html', + styleUrls: ['./emails.component.css'] +}) +export class EmailDialog { + + emailForm = new FormGroup({ + id: new FormControl(''), + description: new FormControl('', [Validators.required]), + subject: new FormControl('', [Validators.required]), + message: new FormControl('', [Validators.required]) + }); + + constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { + this.emailForm.get('id')?.setValue(data.id); + this.emailForm.get('description')?.setValue(data.description); + this.emailForm.get('subject')?.setValue(data.subject); + this.emailForm.get('message')?.setValue(data.message); + } + + onSubmit(): void { + const email = Object.assign({}, this.data, this.emailForm.value); + this.service.saveEmailTemplate(email, (data: void) => this.dialogRef.close(1), this.emailForm); + } + + onNoClick(): void { + this.dialogRef.close(); + } +} diff --git a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html index 416d80dd..d5d864d7 100644 --- a/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html +++ b/frontends/dnet-is-application/src/app/main-menu-panels/main-menu-panels.component.html @@ -51,6 +51,15 @@ + + + Workflows + +
+ Workflows +
+ + Tools diff --git a/frontends/dnet-is-application/src/app/resources/metadata-dialog.html b/frontends/dnet-is-application/src/app/resources/metadata-dialog.html index 1c391d15..c017f64d 100644 --- a/frontends/dnet-is-application/src/app/resources/metadata-dialog.html +++ b/frontends/dnet-is-application/src/app/resources/metadata-dialog.html @@ -13,9 +13,14 @@ + + SubType (optional) + + + Name - + This field is required diff --git a/frontends/dnet-is-application/src/app/resources/new-dialog.html b/frontends/dnet-is-application/src/app/resources/new-dialog.html index 4ed97939..031a4380 100644 --- a/frontends/dnet-is-application/src/app/resources/new-dialog.html +++ b/frontends/dnet-is-application/src/app/resources/new-dialog.html @@ -5,10 +5,15 @@ Name - + This field is required + + SubType (optional) + + + Description @@ -16,7 +21,7 @@ Content ({{data.contentType}}) - + This field is required diff --git a/frontends/dnet-is-application/src/app/resources/resources.component.html b/frontends/dnet-is-application/src/app/resources/resources.component.html index 17ce536b..6463697a 100644 --- a/frontends/dnet-is-application/src/app/resources/resources.component.html +++ b/frontends/dnet-is-application/src/app/resources/resources.component.html @@ -12,14 +12,18 @@ - {{r.name}} {{type.contentType}} + + {{r.name}} + {{r.subtype}} + {{type.contentType}} +

{{r.description}}

- Id: {{r.id}}
Creation date: {{r.creationDate}}
Modification date: - {{r.modificationDate}} + Id: {{r.id}}
+ Creation date: {{r.creationDate}}
+ Modification date: {{r.modificationDate}}

diff --git a/frontends/dnet-is-application/src/app/resources/resources.component.ts b/frontends/dnet-is-application/src/app/resources/resources.component.ts index c801a65a..1d5c2570 100644 --- a/frontends/dnet-is-application/src/app/resources/resources.component.ts +++ b/frontends/dnet-is-application/src/app/resources/resources.component.ts @@ -3,7 +3,7 @@ import { ISService } from '../common/is.service'; import { ActivatedRoute } from '@angular/router'; import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { ResourceType, SimpleResource } from '../common/is.model'; -import { FormControl, FormGroup } from '@angular/forms'; +import { FormControl, FormGroup, Validators } from '@angular/forms'; @Component({ selector: 'app-resources', @@ -115,12 +115,16 @@ export class ResContentDialog { }) export class ResMetadataDialog { metadataForm = new FormGroup({ - name: new FormControl(''), + name: new FormControl('', [Validators.required]), + subtype: new FormControl(''), description: new FormControl('') }); constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { this.metadataForm.get('name')?.setValue(data.name); + if (data.subtype) { + this.metadataForm.get('subtype')?.setValue(data.subtype); + } if (data.description) { this.metadataForm.get('description')?.setValue(data.description); } @@ -143,9 +147,10 @@ export class ResMetadataDialog { }) export class ResCreateNewDialog { newResourceForm = new FormGroup({ - name: new FormControl(''), + name: new FormControl('', [Validators.required]), + subtype: new FormControl(''), description: new FormControl(''), - content: new FormControl('') + content: new FormControl('', [Validators.required]) }); constructor(public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { } @@ -153,10 +158,11 @@ export class ResCreateNewDialog { onSubmit(): void { let name: string = this.newResourceForm.get('name')?.value!; let type: string = this.data.id!; + let subtype: string = this.newResourceForm.get('subtype')?.value!; let description: string = this.newResourceForm.get('description')?.value!; let content: string = this.newResourceForm.get('content')?.value!; - this.service.addSimpleResource(name, type, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm); + this.service.addSimpleResource(name, type, subtype, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm); } onNoClick(): void { this.dialogRef.close(); diff --git a/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts b/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts index 0d37f421..755f8f18 100644 --- a/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts +++ b/frontends/dnet-is-application/src/app/vocabularies/vocabularies.component.ts @@ -137,8 +137,6 @@ export class VocabularyEditorComponent implements OnInit, AfterViewInit { } } - - } @Component({ diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instance.dialog.html b/frontends/dnet-is-application/src/app/wf-instances/wf-instance.dialog.html new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.css b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.css new file mode 100644 index 00000000..e69de29b diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html new file mode 100644 index 00000000..2efe0381 --- /dev/null +++ b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.html @@ -0,0 +1,7 @@ +

Workflow Instances

+ + + Content 1 + Content 2 + Content 3 + diff --git a/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts new file mode 100644 index 00000000..da515868 --- /dev/null +++ b/frontends/dnet-is-application/src/app/wf-instances/wf-instances.component.ts @@ -0,0 +1,10 @@ +import { Component } from '@angular/core'; + +@Component({ + selector: 'app-wf-instances', + templateUrl: './wf-instances.component.html', + styleUrls: ['./wf-instances.component.css'] +}) +export class WfInstancesComponent { + +} diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java index db7f48b1..41adb299 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java @@ -1,7 +1,8 @@ package eu.dnetlib.data.mdstore; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.Comparator; -import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -112,9 +113,9 @@ public class MDStoreService { private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) { final MDStoreVersion v = new MDStoreVersion(); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); - final String versionId = md.getId() + "-" + now.getTime(); + final String versionId = md.getId() + "-" + now.toEpochSecond(ZoneOffset.UTC); v.setId(versionId); v.setMdstore(md.getId()); v.setLastUpdate(null); @@ -189,7 +190,7 @@ public class MDStoreService { mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v)); v.setWriting(false); v.setSize(size); - v.setLastUpdate(new Date()); + v.setLastUpdate(LocalDateTime.now()); mdstoreVersionRepository.save(v); return v; @@ -258,7 +259,7 @@ public class MDStoreService { md.setLayout(layout); md.setType(type); md.setInterpretation(interpretation); - md.setCreationDate(new Date()); + md.setCreationDate(LocalDateTime.now()); md.setDatasourceName(dsName); md.setDatasourceId(dsId); md.setApiId(apiId); diff --git a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java index 66ba910b..5b1a1d84 100644 --- a/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java +++ b/libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/backends/MockBackend.java @@ -1,8 +1,8 @@ package eu.dnetlib.data.mdstore.backends; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -49,8 +49,8 @@ public class MockBackend implements MDStoreBackend { rec.setOriginalId("mck-" + i); rec.setId("mock________::mck-" + i); rec.setBody("" + i + ""); - rec.setDateOfCollection(new Date().getTime()); - rec.setDateOfTransformation(new Date().getTime()); + rec.setDateOfCollection(Instant.now().toEpochMilli()); + rec.setDateOfTransformation(Instant.now().toEpochMilli()); rec.setEncoding("XML"); rec.setProvenance(MOCK_PROVENANCE); list.add(rec); diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java new file mode 100644 index 00000000..b5369e4d --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/common/model/EmailTemplate.java @@ -0,0 +1,61 @@ +package eu.dnetlib.common.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "emails") +public class EmailTemplate implements Serializable { + + private static final long serialVersionUID = -8424437958140000626L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "description") + private String description; + + @Column(name = "subject") + private String subject; + + @Column(name = "message") + private String message; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getSubject() { + return subject; + } + + public void setSubject(final String subject) { + this.subject = subject; + } + + public String getMessage() { + return message; + } + + public void setMessage(final String message) { + this.message = message; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java index 82548da9..ff71cda7 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -12,8 +12,6 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -55,8 +53,7 @@ public class MDStore implements Serializable { private Map params = new LinkedHashMap<>(); @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; + private LocalDateTime creationDate; @Override public int hashCode() { @@ -143,11 +140,11 @@ public class MDStore implements Serializable { this.params = params; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java index d2e07c08..29e958d5 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -10,8 +10,6 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -36,8 +34,7 @@ public class MDStoreVersion implements Serializable { private int readCount = 0; @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; + private LocalDateTime lastUpdate; @Column(name = "size") private long size = 0; @@ -78,11 +75,11 @@ public class MDStoreVersion implements Serializable { this.readCount = readCount; } - public Date getLastUpdate() { + public LocalDateTime getLastUpdate() { return lastUpdate; } - public void setLastUpdate(final Date lastUpdate) { + public void setLastUpdate(final LocalDateTime lastUpdate) { this.lastUpdate = lastUpdate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java index 17bdb4f8..ff570971 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java @@ -1,7 +1,7 @@ package eu.dnetlib.data.mdstore.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -12,8 +12,6 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; @@ -54,12 +52,10 @@ public class MDStoreWithInfo implements Serializable { private String currentVersion; @Column(name = "creation_date") - @Temporal(TemporalType.TIMESTAMP) - private Date creationDate; + private LocalDateTime creationDate; @Column(name = "lastupdate") - @Temporal(TemporalType.TIMESTAMP) - private Date lastUpdate; + private LocalDateTime lastUpdate; @Column(name = "size") private long size = 0; @@ -143,19 +139,19 @@ public class MDStoreWithInfo implements Serializable { this.currentVersion = currentVersion; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } - public Date getLastUpdate() { + public LocalDateTime getLastUpdate() { return lastUpdate; } - public void setLastUpdate(final Date lastUpdate) { + public void setLastUpdate(final LocalDateTime lastUpdate) { this.lastUpdate = lastUpdate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java index 36f357c4..fe882b1a 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Datasource.java @@ -2,6 +2,7 @@ package eu.dnetlib.dsm.model; import java.io.Serializable; import java.sql.Date; +import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; @@ -89,7 +90,7 @@ public class Datasource implements Serializable { private String registeredby; - private Date registrationdate; + private LocalDateTime registrationdate; private String subjects; @@ -99,10 +100,10 @@ public class Datasource implements Serializable { private Boolean consentTermsOfUse; @Column(name = "consenttermsofusedate") - private Date consentTermsOfUseDate; + private LocalDateTime consentTermsOfUseDate; @Column(name = "lastconsenttermsofusedate") - private Date lastConsentTermsOfUseDate; + private LocalDateTime lastConsentTermsOfUseDate; @Column(name = "fulltextdownload") private Boolean fullTextDownload; @@ -401,11 +402,11 @@ public class Datasource implements Serializable { this.registeredby = registeredby; } - public Date getRegistrationdate() { + public LocalDateTime getRegistrationdate() { return registrationdate; } - public void setRegistrationdate(final Date registrationdate) { + public void setRegistrationdate(final LocalDateTime registrationdate) { this.registrationdate = registrationdate; } @@ -433,19 +434,19 @@ public class Datasource implements Serializable { this.consentTermsOfUse = consentTermsOfUse; } - public Date getConsentTermsOfUseDate() { + public LocalDateTime getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public void setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public void setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; } - public Date getLastConsentTermsOfUseDate() { + public LocalDateTime getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public void setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public void setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java index ec5c9e87..25ee8533 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/dsm/model/Organization.java @@ -1,7 +1,7 @@ package eu.dnetlib.dsm.model; import java.io.Serializable; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.Set; import javax.persistence.CascadeType; @@ -34,7 +34,7 @@ public class Organization implements Serializable { private String country; private String collectedfrom; - private Date dateofcollection; + private LocalDateTime dateofcollection; private String provenanceaction; @ManyToMany(cascade = { @@ -98,11 +98,11 @@ public class Organization implements Serializable { this.collectedfrom = collectedfrom; } - public Date getDateofcollection() { + public LocalDateTime getDateofcollection() { return dateofcollection; } - public void setDateofcollection(final Date dateofcollection) { + public void setDateofcollection(final LocalDateTime dateofcollection) { this.dateofcollection = dateofcollection; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java new file mode 100644 index 00000000..bc29ade1 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/errors/WorkflowManagerException.java @@ -0,0 +1,14 @@ +package eu.dnetlib.errors; + +public class WorkflowManagerException extends Exception { + + private static final long serialVersionUID = -9067581185191425823L; + + public WorkflowManagerException(final String message, final Throwable cause) { + super(message, cause); + } + + public WorkflowManagerException(final String message) { + super(message); + } +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java index 60565e0b..6a73a64e 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/is/model/resource/SimpleResource.java @@ -1,14 +1,12 @@ package eu.dnetlib.is.model.resource; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; @Entity @Table(name = "resources") @@ -26,16 +24,17 @@ public class SimpleResource implements Serializable { @Column(name = "type") private String type; + @Column(name = "subtype") + private String subtype; + @Column(name = "description") private String description; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "creation_date") - private Date creationDate; + private LocalDateTime creationDate; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "modification_date") - private Date modificationDate; + private LocalDateTime modificationDate; public String getId() { return id; @@ -61,6 +60,14 @@ public class SimpleResource implements Serializable { this.type = type; } + public String getSubtype() { + return subtype; + } + + public void setSubtype(final String subtype) { + this.subtype = subtype; + } + public String getDescription() { return description; } @@ -69,19 +76,19 @@ public class SimpleResource implements Serializable { this.description = description; } - public Date getCreationDate() { + public LocalDateTime getCreationDate() { return creationDate; } - public void setCreationDate(final Date creationDate) { + public void setCreationDate(final LocalDateTime creationDate) { this.creationDate = creationDate; } - public Date getModificationDate() { + public LocalDateTime getModificationDate() { return modificationDate; } - public void setModificationDate(final Date modificationDate) { + public void setModificationDate(final LocalDateTime modificationDate) { this.modificationDate = modificationDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java similarity index 80% rename from libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java rename to libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java index 0bea598f..cbd78045 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/msro/model/history/WfProcessExecution.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/history/model/WfProcessExecution.java @@ -1,15 +1,13 @@ -package eu.dnetlib.msro.model.history; +package eu.dnetlib.manager.history.model; import java.io.Serializable; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Map; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; @@ -32,6 +30,9 @@ public class WfProcessExecution implements Serializable { @Column(name = "process_id") private String processId; + @Column(name = "wf_instance_id") + private String wfInstanceId; + @Column(name = "name") private String name; @@ -41,13 +42,11 @@ public class WfProcessExecution implements Serializable { @Column(name = "status") private String status; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "start_date") - private Date startDate; + private LocalDateTime startDate; - @Temporal(TemporalType.TIMESTAMP) @Column(name = "end_date") - private Date endDate; + private LocalDateTime endDate; @Column(name = "ds_id") private String dsId; @@ -70,6 +69,14 @@ public class WfProcessExecution implements Serializable { this.processId = processId; } + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + public String getName() { return name; } @@ -94,19 +101,19 @@ public class WfProcessExecution implements Serializable { this.status = status; } - public Date getStartDate() { + public LocalDateTime getStartDate() { return startDate; } - public void setStartDate(final Date startDate) { + public void setStartDate(final LocalDateTime startDate) { this.startDate = startDate; } - public Date getEndDate() { + public LocalDateTime getEndDate() { return endDate; } - public void setEndDate(final Date endDate) { + public void setEndDate(final LocalDateTime endDate) { this.endDate = endDate; } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java new file mode 100644 index 00000000..dbb0c205 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/NotificationCondition.java @@ -0,0 +1,5 @@ +package eu.dnetlib.manager.wf.model; + +public enum NotificationCondition { + ALWAYS, NEVER, ONLY_SUCCESS, ONLY_FAILED +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java new file mode 100644 index 00000000..76fcc054 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowGraph.java @@ -0,0 +1,269 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.env.Environment; + +public class WorkflowGraph implements Serializable { + + private static final long serialVersionUID = 5919290887480115842L; + + public List parameters; + public List graph; + + public List getGraph() { + return graph; + } + + public void setGraph(final List graph) { + this.graph = graph; + } + + public List getParameters() { + return parameters; + } + + public void setParameters(final List parameters) { + this.parameters = parameters; + } + + class WfParam implements Serializable { + + private static final long serialVersionUID = 5885589803738655166L; + + private String name; + private String description; + private String type; + private String defaultValue; + private boolean required; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(final String description) { + this.description = description; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(final String defaultValue) { + this.defaultValue = defaultValue; + } + + public boolean isRequired() { + return required; + } + + public void setRequired(final boolean required) { + this.required = required; + } + + } + + public class Node implements Serializable { + + private static final long serialVersionUID = -3695762832959801906L; + + private static final String regExRef = "\\$\\{(\\w*)\\}"; + + private String name; + private String type; + private boolean isStart = false; + private boolean isJoin = false; + private List arcs; + private List input; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public boolean isStart() { + return isStart; + } + + public void setStart(final boolean isStart) { + this.isStart = isStart; + } + + public boolean isJoin() { + return isJoin; + } + + public void setJoin(final boolean isJoin) { + this.isJoin = isJoin; + } + + public List getArcs() { + return arcs; + } + + public void setArcs(final List arcs) { + this.arcs = arcs; + } + + public List getInput() { + return input; + } + + public void setInput(final List input) { + this.input = input; + } + + public Map findEnvParams() { + return input.stream() + .filter(p -> StringUtils.isNotBlank(p.getEnv())) + .collect(Collectors.toMap(NodeParam::getName, NodeParam::getEnv)); + } + + public Map calculateInitialParams(final Map globalParams, final Environment environment) { + final Map map = new HashMap<>(); + + input.stream() + .filter(p -> StringUtils.isBlank(p.getEnv())) + .forEach(p -> map.put(p.getName(), calculateSimpleValue(p, globalParams, environment))); + + return map; + } + + private Object calculateSimpleValue(final NodeParam p, final Map globalParams, final Environment environment) { + String value = p.getValue(); + final String ref = p.getRef(); + final String prop = p.getProperty(); + + if (StringUtils.isNotBlank(ref) && StringUtils.isNotBlank(globalParams.get(ref))) { + return globalParams.get(ref); + } else if (StringUtils.isNotBlank(value)) { + final Matcher matcher = Pattern.compile(regExRef, Pattern.MULTILINE).matcher(value); + while (matcher.find()) { + final String rName = matcher.group(1); + final String rValue = globalParams.get(rName); + if (StringUtils.isBlank(rValue)) { return null; } + value = value.replaceAll(Pattern.quote(matcher.group(0)), rValue); + System.out.println("NEW VALUE " + value); + } + return value; + } else if (StringUtils.isNotBlank(prop)) { + return environment.getProperty(prop); + } else { + return null; + } + + } + + } + + public class Arc implements Serializable { + + private static final long serialVersionUID = 7866138976929522262L; + + private String to; + private String condition; + + public String getTo() { + return to; + } + + public void setTo(final String to) { + this.to = to; + } + + public String getCondition() { + return condition; + } + + public void setCondition(final String condition) { + this.condition = condition; + } + } + + class NodeParam implements Serializable { + + private static final long serialVersionUID = 7815785723401725707L; + + private String name; + private String value; + private String ref; + private String property; + private String env; + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + + public String getRef() { + return ref; + } + + public void setRef(final String ref) { + this.ref = ref; + } + + public String getProperty() { + return property; + } + + public void setProperty(final String property) { + this.property = property; + } + + public String getEnv() { + return env; + } + + public void setEnv(final String env) { + this.env = env; + } + + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java new file mode 100644 index 00000000..d9478332 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowInstance.java @@ -0,0 +1,232 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Transient; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; +import com.vladmihalcea.hibernate.type.json.JsonStringType; + +@Entity +@Table(name = "workflow_instances") +@TypeDefs({ + @TypeDef(name = "json", typeClass = JsonStringType.class), + @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) +}) +public class WorkflowInstance implements Serializable { + + private static final long serialVersionUID = 7503841966138333044L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "name") + private String name; + + @Column(name = "section") + private String section; + + @Type(type = "jsonb") + @Column(name = "details", columnDefinition = "jsonb") + private Map details = new LinkedHashMap<>(); + + @Column(name = "priority") + private Integer priority; + + @Column(name = "dsid") + private String dsId; + + @Column(name = "dsname") + private String dsName; + + @Column(name = "apiid") + private String apiId; + + @Column(name = "enabled") + private boolean enabled; + + @Column(name = "configured") + private boolean configured; + + @Column(name = "scheduling_enabled") + private boolean schedulingEnabled; + + @Column(name = "scheduling_cron") + private String cronExpression; + + @Column(name = "scheduling_min_interval") + private int cronMinInterval; + + @Column(name = "workflow") + private String workflow; + + @Column(name = "destroy_wf") + private String destroyWf; + + @Type(type = "jsonb") + @Column(name = "system_params", columnDefinition = "jsonb") + private Map systemParams = new LinkedHashMap<>(); + + @Type(type = "jsonb") + @Column(name = "user_params", columnDefinition = "jsonb") + private Map userParams = new LinkedHashMap<>(); + + @Transient + private String parentId; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getSection() { + return section; + } + + public void setSection(final String section) { + this.section = section; + } + + public Map getDetails() { + return details; + } + + public void setDetails(final Map details) { + this.details = details; + } + + public Integer getPriority() { + return priority; + } + + public void setPriority(final Integer priority) { + this.priority = priority; + } + + public String getDsId() { + return dsId; + } + + public void setDsId(final String dsId) { + this.dsId = dsId; + } + + public String getDsName() { + return dsName; + } + + public void setDsName(final String dsName) { + this.dsName = dsName; + } + + public String getApiId() { + return apiId; + } + + public void setApiId(final String apiId) { + this.apiId = apiId; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(final boolean enabled) { + this.enabled = enabled; + } + + public boolean isConfigured() { + return configured; + } + + public void setConfigured(final boolean configured) { + this.configured = configured; + } + + public boolean isSchedulingEnabled() { + return schedulingEnabled; + } + + public void setSchedulingEnabled(final boolean schedulingEnabled) { + this.schedulingEnabled = schedulingEnabled; + } + + public String getCronExpression() { + return cronExpression; + } + + public void setCronExpression(final String cronExpression) { + this.cronExpression = cronExpression; + } + + public int getCronMinInterval() { + return cronMinInterval; + } + + public void setCronMinInterval(final int cronMinInterval) { + this.cronMinInterval = cronMinInterval; + } + + public String getWorkflow() { + return workflow; + } + + public void setWorkflow(final String workflow) { + this.workflow = workflow; + } + + public String getDestroyWf() { + return destroyWf; + } + + public void setDestroyWf(final String destroyWf) { + this.destroyWf = destroyWf; + } + + public Map getSystemParams() { + return systemParams; + } + + public void setSystemParams(final Map systemParams) { + this.systemParams = systemParams; + } + + public Map getUserParams() { + return userParams; + } + + public void setUserParams(final Map userParams) { + this.userParams = userParams; + } + + public String getParentId() { + return parentId; + } + + public void setParentId(final String parentId) { + this.parentId = parentId; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java new file mode 100644 index 00000000..9e0f177f --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowParamDesc.java @@ -0,0 +1,5 @@ +package eu.dnetlib.manager.wf.model; + +public class WorkflowParamDesc { + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java new file mode 100644 index 00000000..9442e279 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscription.java @@ -0,0 +1,68 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; + +@Entity +@Table(name = "workflow_subscriptions") +@IdClass(WorkflowSubscriptionPK.class) +public class WorkflowSubscription implements Serializable { + + private static final long serialVersionUID = -3662770213782581404L; + + @Id + @Column(name = "wf_instance_id") + private String wfInstanceId; + + @Id + @Column(name = "condition") + @Enumerated(EnumType.STRING) + private NotificationCondition condition; + + @Id + @Column(name = "email") + private String email; + + @Column(name = "message_id") + private String messageId; + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + + public NotificationCondition getCondition() { + return condition; + } + + public void setCondition(final NotificationCondition condition) { + this.condition = condition; + } + + public String getEmail() { + return email; + } + + public void setEmail(final String email) { + this.email = email; + } + + public String getMessageId() { + return messageId; + } + + public void setMessageId(final String messageId) { + this.messageId = messageId; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java new file mode 100644 index 00000000..61b3abfc --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/manager/wf/model/WorkflowSubscriptionPK.java @@ -0,0 +1,39 @@ +package eu.dnetlib.manager.wf.model; + +import java.io.Serializable; + +public class WorkflowSubscriptionPK implements Serializable { + + private static final long serialVersionUID = -7569690774071644848L; + + private String wfInstanceId; + + private NotificationCondition condition; + + private String email; + + public String getWfInstanceId() { + return wfInstanceId; + } + + public void setWfInstanceId(final String wfInstanceId) { + this.wfInstanceId = wfInstanceId; + } + + public NotificationCondition getCondition() { + return condition; + } + + public void setCondition(final NotificationCondition condition) { + this.condition = condition; + } + + public String getEmail() { + return email; + } + + public void setEmail(final String email) { + this.email = email; + } + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java new file mode 100644 index 00000000..046e6c6d --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/CountedValue.java @@ -0,0 +1,13 @@ +package eu.dnetlib.utils; + +public interface CountedValue { + + public String getValue(); + + public void setValue(); + + public long getCount(); + + public void setCount(final long count); + +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java index 63691bf7..3cdda237 100644 --- a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java @@ -1,9 +1,11 @@ package eu.dnetlib.utils; import java.io.StringWriter; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Locale; +import java.util.TimeZone; public class DateUtils { @@ -12,7 +14,7 @@ public class DateUtils { private static final long HOUR = MINUTE * 60; private static final long DAY = HOUR * 24; private static final long YEAR = DAY * 365; - private static final SimpleDateFormat ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US); + private static final DateTimeFormatter ISO8601FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault()); public static String elapsedTime(long t) { final StringWriter a = new StringWriter(); @@ -78,7 +80,13 @@ public class DateUtils { } public static String calculate_ISO8601(final long l) { - String result = ISO8601FORMAT.format(new Date(l)); + + final LocalDateTime time = + LocalDateTime.ofInstant(Instant.ofEpochMilli(l), TimeZone + .getDefault() + .toZoneId()); + + String result = time.format(ISO8601FORMAT); // convert YYYYMMDDTHH:mm:ss+HH00 into YYYYMMDDTHH:mm:ss+HH:00 // - note the added colon for the Timezone result = result.substring(0, result.length() - 2) + ":" + result.substring(result.length() - 2); @@ -89,4 +97,8 @@ public class DateUtils { return Long.toString(Math.floorDiv(d, n)); } + public static long now() { + return Instant.now().toEpochMilli(); + } + } diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java new file mode 100644 index 00000000..dd1fb957 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/Stoppable.java @@ -0,0 +1,10 @@ +package eu.dnetlib.utils; + +public interface Stoppable { + + void stop(); + + void resume(); + + public StoppableDetails getStopDetails(); +} diff --git a/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java new file mode 100644 index 00000000..708198c0 --- /dev/null +++ b/libs/dnet-is-common/src/main/java/eu/dnetlib/utils/StoppableDetails.java @@ -0,0 +1,33 @@ +package eu.dnetlib.utils; + +public class StoppableDetails { + + public enum StopStatus { + STOPPED, + STOPPING, + RUNNING + }; + + private final String title; + private final String description; + private final StopStatus status; + + public StoppableDetails(final String title, final String description, final StopStatus status) { + this.title = title; + this.description = description; + this.status = status; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public StopStatus getStatus() { + return status; + } + +} diff --git a/libs/dnet-is-common/src/main/resources/sql/schema.sql b/libs/dnet-is-common/src/main/resources/sql/schema.sql index a17f71f8..041c22ab 100644 --- a/libs/dnet-is-common/src/main/resources/sql/schema.sql +++ b/libs/dnet-is-common/src/main/resources/sql/schema.sql @@ -105,16 +105,17 @@ CREATE INDEX ON context_cat_concepts_lvl_2 (parent); -- WF History CREATE TABLE wf_history ( - process_id text PRIMARY KEY, - name text NOT NULL, - family text NOT NULL, - status text NOT NULL, - start_date timestamp NOT NULL, - end_date timestamp NOT NULL, - ds_id text, - ds_name text, - ds_api text, - details jsonb + process_id text PRIMARY KEY, + wf_instance_id text NOT NULL, + name text NOT NULL, + family text NOT NULL, + status text NOT NULL, + start_date timestamp NOT NULL, + end_date timestamp NOT NULL, + ds_id text, + ds_name text, + ds_api text, + details jsonb ); -- Other Resources @@ -130,7 +131,8 @@ INSERT INTO resource_types(id, name, content_type) VALUES ('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'), ('cleaning_rule', 'Cleaning Rules', 'application/xml'), ('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml') - ('dedup_configuration', 'Dedup Configurations', 'application/json'); + ('dedup_configuration', 'Dedup Configurations', 'application/json') + ('workflow', 'Workflows', 'application/json'); CREATE TABLE resources ( id text PRIMARY KEY, @@ -138,6 +140,7 @@ CREATE TABLE resources ( description text, content text NOT NULL DEFAULT '', type text NOT NULL REFERENCES resource_types(id), + subtype text, creation_date timestamp NOT NULL DEFAULT now(), modification_date timestamp NOT NULL DEFAULT now() ); @@ -177,6 +180,14 @@ CREATE VIEW resource_types_view AS ( count(*) AS count, false AS simple FROM protocols +) UNION ALL ( + SELECT + 'email' AS id, + 'Email templates' AS name, + 'text/plain' AS content_type, + count(*) AS count, + false AS simple + FROM emails ); CREATE TABLE mdstores ( @@ -241,3 +252,40 @@ GROUP BY md.id, v1.lastupdate, v1.size; +-- Email Templates +CREATE TABLE emails ( + id text PRIMARY KEY, + description text NOT NULL, + subject text NOT NULL, + message text NOT NULL +); + +-- Workflows + +CREATE TABLE workflow_instances ( + id text PRIMARY KEY, + name text NOT NULL, + section text, + details jsonb NOT NULL DEFAULT '{}', + priority int, + dsid text, + dsname text, + apiid text, + enabled boolean NOT NULL DEFAULT false, + configured boolean NOT NULL DEFAULT false, + scheduling_enabled boolean NOT NULL DEFAULT false, + scheduling_cron text, + scheduling_min_interval int, + workflow text REFERENCES resource(id), + destroy_wf text REFERENCES resource(id), + system_params jsonb NOT NULL DEFAULT '{}', + user_params jsonb NOT NULL DEFAULT '{}' +); + +CREATE TABLE workflow_subscriptions ( + wf_instance_id text NOT NULL REFERENCES workflow_instances(id), + condition text NOT NULL, + email text NOT NULL, + message_id text NOT NULL REFERENCES emails(id), + PRIMARY KEY (wf_instance_id, condition, email) +); diff --git a/libs/dnet-is-services/pom.xml b/libs/dnet-is-services/pom.xml index a21a4f9d..709c81b9 100644 --- a/libs/dnet-is-services/pom.xml +++ b/libs/dnet-is-services/pom.xml @@ -24,6 +24,11 @@ jaxen jaxen + + + org.springframework.boot + spring-boot-starter-thymeleaf + diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java new file mode 100644 index 00000000..f1ce156b --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/common/repository/EmailTemplateRepository.java @@ -0,0 +1,9 @@ +package eu.dnetlib.common.repository; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.common.model.EmailTemplate; + +public interface EmailTemplateRepository extends JpaRepository { + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java index 5f292ce1..f9a1c8a3 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java @@ -9,7 +9,7 @@ import static eu.dnetlib.dsm.utils.DsmMappingUtils.createId; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.sql.Date; +import java.time.LocalDate; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -368,7 +368,7 @@ public class DsmService { private void ensureRegistrationDate(final String dsId) { if (!dsRepository.hasRegistrationdate(dsId)) { log.info("setting registration date for datasource: " + dsId); - dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis())); + dsRepository.setRegistrationDate(dsId, LocalDate.now()); } } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java index 6bf9c977..8e12cb5c 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/ApiDetails.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.Set; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -27,15 +27,15 @@ public class ApiDetails extends ApiIgnoredProperties { private Integer lastCollectionTotal; - private Date lastCollectionDate; + private LocalDateTime lastCollectionDate; private Integer lastAggregationTotal; - private Date lastAggregationDate; + private LocalDateTime lastAggregationDate; private Integer lastDownloadTotal; - private Date lastDownloadDate; + private LocalDateTime lastDownloadDate; private String baseurl; @@ -71,7 +71,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastCollectionTotal; } - public Date getLastCollectionDate() { + public LocalDateTime getLastCollectionDate() { return lastCollectionDate; } @@ -79,7 +79,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastAggregationTotal; } - public Date getLastAggregationDate() { + public LocalDateTime getLastAggregationDate() { return lastAggregationDate; } @@ -87,7 +87,7 @@ public class ApiDetails extends ApiIgnoredProperties { return lastDownloadTotal; } - public Date getLastDownloadDate() { + public LocalDateTime getLastDownloadDate() { return lastDownloadDate; } @@ -125,7 +125,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastCollectionDate(final Date lastCollectionDate) { + public ApiDetails setLastCollectionDate(final LocalDateTime lastCollectionDate) { this.lastCollectionDate = lastCollectionDate; return this; } @@ -135,7 +135,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastAggregationDate(final Date lastAggregationDate) { + public ApiDetails setLastAggregationDate(final LocalDateTime lastAggregationDate) { this.lastAggregationDate = lastAggregationDate; return this; } @@ -145,7 +145,7 @@ public class ApiDetails extends ApiIgnoredProperties { return this; } - public ApiDetails setLastDownloadDate(final Date lastDownloadDate) { + public ApiDetails setLastDownloadDate(final LocalDateTime lastDownloadDate) { this.lastDownloadDate = lastDownloadDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java index 096a76dd..ef7c0a08 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetails.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import java.util.Set; import javax.persistence.Transient; @@ -48,12 +48,12 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { private String languages; - private Date dateofvalidation; + private LocalDate dateofvalidation; @NotBlank private String eoscDatasourceType; - private Date dateofcollection; + private LocalDate dateofcollection; private String platform; @@ -82,9 +82,9 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { private Boolean fullTextDownload; - private Date consentTermsOfUseDate; + private LocalDate consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDate lastConsentTermsOfUseDate; private Set organizations; @@ -95,7 +95,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { @Deprecated private String typology; - private Date registrationdate; + private LocalDate registrationdate; public String getId() { return id; @@ -145,7 +145,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return namespaceprefix; } - public Date getDateofvalidation() { + public LocalDate getDateofvalidation() { return dateofvalidation; } @@ -153,7 +153,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return eoscDatasourceType; } - public Date getDateofcollection() { + public LocalDate getDateofcollection() { return dateofcollection; } @@ -277,7 +277,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public DatasourceDetails setDateofvalidation(final Date dateofvalidation) { + public DatasourceDetails setDateofvalidation(final LocalDate dateofvalidation) { this.dateofvalidation = dateofvalidation; return this; } @@ -287,7 +287,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public DatasourceDetails setDateofcollection(final Date dateofcollection) { + public DatasourceDetails setDateofcollection(final LocalDate dateofcollection) { this.dateofcollection = dateofcollection; return this; } @@ -367,11 +367,11 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDate getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceDetails setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -396,20 +396,20 @@ public class DatasourceDetails extends DatasourceIgnoredProperties { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDate getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceDetails setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } - public Date getRegistrationdate() { + public LocalDate getRegistrationdate() { return registrationdate; } - public DatasourceDetails setRegistrationdate(final Date registrationdate) { + public DatasourceDetails setRegistrationdate(final LocalDate registrationdate) { this.registrationdate = registrationdate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java index e9fe39f8..3451ba70 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceDetailsUpdate.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import java.util.Set; import javax.validation.constraints.Email; @@ -57,9 +57,9 @@ public class DatasourceDetailsUpdate { private Boolean consentTermsOfUse; - private Date consentTermsOfUseDate; + private LocalDate consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDate lastConsentTermsOfUseDate; private Boolean fullTextDownload; @@ -207,11 +207,11 @@ public class DatasourceDetailsUpdate { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDate getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceDetailsUpdate setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -225,11 +225,11 @@ public class DatasourceDetailsUpdate { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDate getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java index 4a1a3e10..dfea4420 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceIgnoredProperties.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.sql.Date; +import java.time.LocalDate; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -13,10 +13,10 @@ public abstract class DatasourceIgnoredProperties { protected String provenanceaction; @JsonIgnore - protected Date releasestartdate; + protected LocalDate releasestartdate; @JsonIgnore - protected Date releaseenddate; + protected LocalDate releaseenddate; @JsonIgnore protected String missionstatementurl; @@ -76,19 +76,19 @@ public abstract class DatasourceIgnoredProperties { this.provenanceaction = provenanceaction; } - public Date getReleasestartdate() { + public LocalDate getReleasestartdate() { return releasestartdate; } - public void setReleasestartdate(final Date releasestartdate) { + public void setReleasestartdate(final LocalDate releasestartdate) { this.releasestartdate = releasestartdate; } - public Date getReleaseenddate() { + public LocalDate getReleaseenddate() { return releaseenddate; } - public void setReleaseenddate(final Date releaseenddate) { + public void setReleaseenddate(final LocalDate releaseenddate) { this.releaseenddate = releaseenddate; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java index b44f19d8..adf99be5 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/DatasourceSnippetExtended.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Set; import javax.validation.constraints.Email; @@ -28,7 +28,7 @@ public class DatasourceSnippetExtended { @Email private String registeredby; - private Date registrationdate; + private LocalDateTime registrationdate; private String eoscDatasourceType; @@ -38,9 +38,9 @@ public class DatasourceSnippetExtended { private Boolean consentTermsOfUse; - private Date consentTermsOfUseDate; + private LocalDateTime consentTermsOfUseDate; - private Date lastConsentTermsOfUseDate; + private LocalDateTime lastConsentTermsOfUseDate; private Boolean fullTextDownload; @@ -89,11 +89,11 @@ public class DatasourceSnippetExtended { this.registeredby = registeredby; } - public Date getRegistrationdate() { + public LocalDateTime getRegistrationdate() { return registrationdate; } - public void setRegistrationdate(final Date registrationdate) { + public void setRegistrationdate(final LocalDateTime registrationdate) { this.registrationdate = registrationdate; } @@ -118,11 +118,11 @@ public class DatasourceSnippetExtended { return this; } - public Date getConsentTermsOfUseDate() { + public LocalDateTime getConsentTermsOfUseDate() { return consentTermsOfUseDate; } - public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { + public DatasourceSnippetExtended setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) { this.consentTermsOfUseDate = consentTermsOfUseDate; return this; } @@ -170,11 +170,11 @@ public class DatasourceSnippetExtended { return this; } - public Date getLastConsentTermsOfUseDate() { + public LocalDateTime getLastConsentTermsOfUseDate() { return lastConsentTermsOfUseDate; } - public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { + public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) { this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; return this; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java index 2a87622f..be9b507a 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/domain/OrganizationIgnoredProperties.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.domain; -import java.util.Date; +import java.time.LocalDateTime; import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -14,7 +14,7 @@ public class OrganizationIgnoredProperties { protected String collectedfrom; @JsonIgnore - protected Date dateofcollection; + protected LocalDateTime dateofcollection; @JsonIgnore protected String provenanceaction; @@ -38,11 +38,11 @@ public class OrganizationIgnoredProperties { this.collectedfrom = collectedfrom; } - public Date getDateofcollection() { + public LocalDateTime getDateofcollection() { return dateofcollection; } - public void setDateofcollection(final Date dateofcollection) { + public void setDateofcollection(final LocalDateTime dateofcollection) { this.dateofcollection = dateofcollection; } diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java index b7319c14..c426dee0 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/repository/DatasourceRepository.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.repository; -import java.sql.Date; +import java.time.LocalDate; import java.util.Optional; import javax.transaction.Transactional; @@ -64,7 +64,7 @@ public interface DatasourceRepository extends JpaRepository, @Modifying @Transactional @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1") - void setRegistrationDate(String id, Date registrationdate); + void setRegistrationDate(String id, LocalDate registrationdate); @Modifying @Transactional diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java index efe75a43..666842ce 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/utils/DsmMappingUtils.java @@ -1,6 +1,6 @@ package eu.dnetlib.dsm.utils; -import java.sql.Date; +import java.time.LocalDateTime; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -98,7 +98,7 @@ public class DsmMappingUtils { final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); o.setId(prefix + ID_SEPARATOR + o.getLegalname()); if (o.getDateofcollection() == null) { - o.setDateofcollection(new Date(System.currentTimeMillis())); + o.setDateofcollection(LocalDateTime.now()); } o.setCollectedfrom(dbe.getCollectedfrom()); }); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java index 10da2b98..993c9fed 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java @@ -1,6 +1,6 @@ package eu.dnetlib.is.resource; -import java.util.Date; +import java.time.LocalDateTime; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -60,17 +60,19 @@ public class SimpleResourceService { @Transactional public SimpleResource saveNewResource(final String name, final String type, + final String subtype, final String description, final String content) throws InformationServiceException { resourceValidator.validate(type, content); - final Date now = new Date(); + final LocalDateTime now = LocalDateTime.now(); final SimpleResource res = new SimpleResource(); res.setId(UUID.randomUUID().toString()); res.setName(name); res.setType(type); + res.setSubtype(subtype); res.setDescription(description); res.setCreationDate(now); res.setModificationDate(now); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java new file mode 100644 index 00000000..6212f9a7 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/WorkflowLogger.java @@ -0,0 +1,51 @@ +package eu.dnetlib.manager.history; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.TimeZone; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.history.model.WfProcessExecution; +import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository; + +@Service +public class WorkflowLogger { + + @Autowired + private WfProcessExecutionRepository wfProcessExecutionRepository; + + public List history(final int total, final Long from, final Long to) { + if (from == null && to == null) { + return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList(); + } else { + + final LocalDateTime fromTime = from != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(from), TimeZone + .getDefault() + .toZoneId()) : LocalDateTime.MIN; + final LocalDateTime toTime = to != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(to), TimeZone + .getDefault() + .toZoneId()) : LocalDateTime.MAX; + + return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(fromTime, toTime); + } + } + + public WfProcessExecution getProcessExecution(final String processId) { + return wfProcessExecutionRepository.findById(processId).get(); + } + + public void saveProcessExecution(final WfProcessExecution pe) { + wfProcessExecutionRepository.save(pe); + } + + public Optional getLastExecutionForInstance(final String id) { + return wfProcessExecutionRepository.findOneByWfInstanceIdOrderByEndDateAsc(id); + } + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java new file mode 100644 index 00000000..0ea555d0 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/history/repository/WfProcessExecutionRepository.java @@ -0,0 +1,16 @@ +package eu.dnetlib.manager.history.repository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.history.model.WfProcessExecution; + +public interface WfProcessExecutionRepository extends JpaRepository { + + List findByEndDateBetweenOrderByEndDateDesc(LocalDateTime start, LocalDateTime end); + + Optional findOneByWfInstanceIdOrderByEndDateAsc(String id); +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java new file mode 100644 index 00000000..1bab126e --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowInstanceRepository.java @@ -0,0 +1,21 @@ +package eu.dnetlib.manager.wf.repository; + +import java.util.stream.Stream; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; + +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.utils.CountedValue; + +public interface WorkflowInstanceRepository extends JpaRepository { + + @Query(value = "select section as value, count(*) as count " + + "from workflow_instances " + + "group by section " + + "order by count desc;", nativeQuery = true) + Stream streamSections(); + + Stream findBySection(String section); + +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java new file mode 100644 index 00000000..ff493ca7 --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/manager/wf/repository/WorkflowSubscriptionRepository.java @@ -0,0 +1,13 @@ +package eu.dnetlib.manager.wf.repository; + +import java.util.List; + +import org.springframework.data.jpa.repository.JpaRepository; + +import eu.dnetlib.manager.wf.model.WorkflowSubscription; +import eu.dnetlib.manager.wf.model.WorkflowSubscriptionPK; + +public interface WorkflowSubscriptionRepository extends JpaRepository { + + List findByWfInstanceId(String wfInstanceId); +} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java deleted file mode 100644 index be3a8c9a..00000000 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/msro/history/repository/WfProcessExecutionRepository.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.dnetlib.msro.history.repository; - -import java.util.Date; -import java.util.List; - -import org.springframework.data.jpa.repository.JpaRepository; - -import eu.dnetlib.msro.model.history.WfProcessExecution; - -public interface WfProcessExecutionRepository extends JpaRepository { - - List findByEndDateBetweenOrderByEndDateDesc(Date start, Date end); -} diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java new file mode 100644 index 00000000..176b2b0d --- /dev/null +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/notifications/mail/EmailService.java @@ -0,0 +1,170 @@ +package eu.dnetlib.notifications.mail; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import javax.annotation.PostConstruct; +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.common.base.Splitter; + +import eu.dnetlib.common.model.EmailTemplate; +import eu.dnetlib.common.repository.EmailTemplateRepository; + +@Service +public class EmailService { + + private static final Log log = LogFactory.getLog(EmailService.class); + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + + @Value("${dnet.configuration.mail.sender.email}") + private String from; + @Value("${dnet.configuration.mail.sender.name}") + private String fromName; + @Value("${dnet.configuration.mail.cc}") + private String cc; + @Value("${dnet.configuration.mail.smtp.host}") + private String smtpHost; + @Value("${dnet.configuration.mail.smtp.port}") + private final int smtpPort = 587; + @Value("${dnet.configuration.mail.smtp.user}") + private String smtpUser; + @Value("${dnet.configuration.mail.smtp.password}") + private String smtpPassword; + @Value("${server.public_url}") + private String baseUrl; + @Value("${dnet.configuration.infrastructure}") + private String infrastructure; + + @Autowired + private EmailTemplateRepository emailTemplateRepository; + + @PostConstruct + private void init() { + new Thread(() -> { + while (true) { + try { + final Message message = this.queue.take(); + if (message != null) { + try { + log.info("Sending mail..."); + Transport.send(message); + log.info("...sent"); + } catch (final MessagingException e) { + log.error("Error sending email", e); + this.queue.add(message); + } + } + } catch (final InterruptedException e1) { + throw new RuntimeException(e1); + } + } + }).start(); + } + + public List listEmailTemplates() { + return emailTemplateRepository.findAll(); + } + + public void saveEmailTemplate(final EmailTemplate email) { + if (StringUtils.isBlank(email.getId()) || email.getId().length() < 10) { + email.setId("email-" + UUID.randomUUID()); + log.info("Saving new email with id: " + email.getId()); + } + emailTemplateRepository.save(email); + } + + public void deleteEmailTemplate(final String id) { + emailTemplateRepository.deleteById(id); + } + + public void sendMail(final String to, final String subject, final String message) { + try { + final Session session = Session.getInstance(obtainProperties(), obtainAuthenticator()); + + final MimeMessage mimeMessage = new MimeMessage(session); + mimeMessage.setFrom(new InternetAddress(this.from, this.fromName)); + mimeMessage.setSubject(subject); + mimeMessage.setContent(message, "text/html; charset=utf-8"); + mimeMessage.setSentDate(new Date()); + + mimeMessage.addRecipient(Message.RecipientType.TO, new InternetAddress(to)); + + if (this.cc != null && !this.cc.isEmpty()) { + for (final String aCC : Splitter.on(",").omitEmptyStrings().trimResults().split(cc)) { + mimeMessage.addRecipient(Message.RecipientType.CC, new InternetAddress(aCC)); + } + } + + this.queue.add(mimeMessage); + + log.info("Mail to " + to + " in queue"); + } catch (final Exception e) { + log.error("Error sending mail", e); + } + } + + public void sendStoredMail(final String to, final String emailId, final Map params) { + + // TODO use a real template library + emailTemplateRepository.findById(emailId).ifPresent(tmpl -> { + String msg = tmpl.getMessage(); + String subject = tmpl.getSubject(); + + for (final Entry e : params.entrySet()) { + msg = msg.replaceAll("{" + e.getKey() + "}", e.getValue().toString()); + subject = subject.replaceAll("{" + e.getKey() + "}", e.getValue().toString()); + } + sendMail(to, subject, msg); + }); + + } + + private Properties obtainProperties() { + final Properties props = new Properties(); + props.put("mail.transport.protocol", "smtp"); + props.put("mail.smtp.host", this.smtpHost); + props.put("mail.smtp.port", this.smtpPort); + props.put("mail.smtp.auth", Boolean.toString(this.smtpUser != null && !this.smtpUser.isEmpty())); + return props; + } + + private Authenticator obtainAuthenticator() { + if (this.smtpUser == null || this.smtpUser.isEmpty()) { return null; } + + final String user = this.smtpUser; + final String passwd = this.smtpPassword; + + return new Authenticator() { + + private final PasswordAuthentication authentication = new PasswordAuthentication(user, passwd); + + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return this.authentication; + } + + }; + } + +} diff --git a/libs/dnet-wf-service/pom.xml b/libs/dnet-wf-service/pom.xml new file mode 100644 index 00000000..6a398249 --- /dev/null +++ b/libs/dnet-wf-service/pom.xml @@ -0,0 +1,48 @@ + + + + eu.dnetlib.dhp + libs + 3.3.3-SNAPSHOT + ../ + + + 4.0.0 + dnet-wf-service + jar + + + + eu.dnetlib.dhp + dnet-is-services + ${project.version} + + + + eu.dnetlib.dhp + dnet-data-services + ${project.version} + + + + + org.junit.jupiter + junit-jupiter + test + + + + org.mockito + mockito-core + test + + + + org.mockito + mockito-junit-jupiter + test + + + + + diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java new file mode 100644 index 00000000..5301a8b0 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/NodeInfo.java @@ -0,0 +1,26 @@ +package eu.dnetlib.manager.wf; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class NodeInfo { + + private String name; + private Map params = new LinkedHashMap<>(); + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public Map getParams() { + return params; + } + + public void setParams(final Map params) { + this.params = params; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java new file mode 100644 index 00000000..8955711d --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WfManager.java @@ -0,0 +1,20 @@ +package eu.dnetlib.manager.wf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Service; + +@Service +public class WfManager { + + @Autowired + private ApplicationContext applicationContext; + + /* + * public Object obtainBaseNode(final String name) { return applicationContext.getBean(name); } + * + * public NodeInfo obtainInfo(final BaseNode node) { final NodeInfo info = new NodeInfo(); final WfNode annotation = + * node.getClass().getAnnotation(WfNode.class); info.setName(annotation.name()); for (final WfParam p : annotation.inputParams()) { + * info.getParams().put(p.name(), p.type()); } return info; } + */ +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java new file mode 100644 index 00000000..145b9949 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -0,0 +1,198 @@ +package eu.dnetlib.manager.wf; + +import java.util.HashMap; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.annotation.PostConstruct; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dsm.DsmService; +import eu.dnetlib.errors.DsmException; +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.is.resource.repository.SimpleResourceRepository; +import eu.dnetlib.manager.wf.model.WorkflowGraph; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.procs.ProcessEngine; +import eu.dnetlib.manager.wf.workflows.procs.ProcessFactory; +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; +import eu.dnetlib.utils.CountedValue; +import eu.dnetlib.utils.Stoppable; +import eu.dnetlib.utils.StoppableDetails; + +@Service +public class WorkflowManagerService implements Stoppable { + + private static final Log log = LogFactory.getLog(WorkflowManagerService.class); + + @Autowired + private ProcessRegistry processRegistry; + @Autowired + private ProcessFactory processFactory; + @Autowired + private ProcessEngine processEngine; + @Autowired + private DsmService dsmService; + + @Autowired + private SimpleResourceRepository simpleResourceRepository; + @Autowired + private WorkflowInstanceRepository workflowInstanceRepository; + + private boolean paused = false; + + @PostConstruct + public void init() { + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { + if (isPaused() || processRegistry.countRunningWfs() >= WorkflowsConstants.MAX_RUNNING_PROCS_SIZE) { return; } + + final WorkflowProcess process = processRegistry.nextProcessToStart(); + if (process != null) { + processEngine.startProcess(process); + } else { + log.debug("WorkflowProcess queue is empty"); + } + }, 10, 10, TimeUnit.SECONDS); + } + + public WorkflowInstance findWorkflowInstance(final String id) throws WorkflowManagerException { + return workflowInstanceRepository.findById(id).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + id)); + } + + public String startRepoHiWorkflow(final String wfId, + final String dsId, + final String apiId, + final ExecutionCallback callback) + throws WorkflowManagerException { + + if (isPaused()) { + log.warn("Wf " + wfId + " not launched, because WorkflowExecutor is preparing for shutdown"); + throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown"); + } + + try { + final String dsName = dsmService.getDs(dsId).getOfficialname(); + + final WorkflowInstance instance = new WorkflowInstance(); + instance.setId("REPO_HI_" + UUID.randomUUID()); + instance.setDetails(new HashMap<>()); + instance.setPriority(100); + instance.setDsId(dsId); + instance.setDsName(dsName); + instance.setApiId(apiId); + instance.setEnabled(true); + instance.setConfigured(true); + instance.setSchedulingEnabled(false); + instance.setCronExpression(""); + instance.setCronMinInterval(0); + instance.setWorkflow(wfId); + instance.setDestroyWf(null); + instance.setSystemParams(new HashMap<>()); + instance.setUserParams(new HashMap<>()); + + return startWorkflowInstance(instance, callback); + } catch (final DsmException e) { + throw new WorkflowManagerException("Invalid datasource: " + dsId, e); + } + } + + public String startWorkflowInstance(final String wfInstanceId, + final String parent, + final ExecutionCallback callback) throws Exception { + + if (isPaused()) { + log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown"); + throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown"); + } + + final WorkflowInstance instance = findWorkflowInstance(wfInstanceId); + + return startWorkflowInstance(instance, callback); + } + + public String startWorkflowInstance(final WorkflowInstance wfInstance, + final ExecutionCallback callback) + throws WorkflowManagerException { + + if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) { + log.warn("Wf instance " + wfInstance.getId() + " is not ready to start"); + throw new WorkflowManagerException("Wf instance " + wfInstance.getId() + " is not ready to start"); + } + + final SimpleResource wfMetadata = simpleResourceRepository + .findById(wfInstance.getWorkflow()) + .filter(r -> r.getType().equals("workflows")) + .orElseThrow(() -> new WorkflowManagerException("WF not found: " + wfInstance.getWorkflow())); + + final WorkflowGraph wfGraph = simpleResourceRepository.findContentById(wfMetadata.getId()) + .map(s -> { + try { + return new ObjectMapper().readValue(s, WorkflowGraph.class); + } catch (final Exception e) { + return (WorkflowGraph) null; + } + }) + .filter(Objects::nonNull) + .orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId())); + + final WorkflowProcess process = + processFactory.newProcess(wfMetadata, wfGraph, wfInstance, callback); + + return processRegistry.registerProcess(process, wfInstance); + } + + @Override + public void stop() { + paused = true; + } + + @Override + public void resume() { + paused = false; + } + + @Override + public StoppableDetails getStopDetails() { + final int count = processRegistry.countRunningWfs(); + + final StoppableDetails.StopStatus status = + isPaused() ? count == 0 ? StoppableDetails.StopStatus.STOPPED : StoppableDetails.StopStatus.STOPPING : StoppableDetails.StopStatus.RUNNING; + + return new StoppableDetails("D-NET workflow manager", "Running workflows: " + count, status); + } + + public ProcessRegistry getProcessRegistry() { + return processRegistry; + } + + public boolean isPaused() { + return paused; + } + + public void setPaused(final boolean paused) { + this.paused = paused; + } + + public Stream streamSections() { + return workflowInstanceRepository.streamSections(); + } + + public Stream streamWfInstancesBySection(final String section) { + return workflowInstanceRepository.findBySection(section); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java new file mode 100644 index 00000000..ee5f811f --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/StreamMimeType.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.annotations; + +public enum StreamMimeType { + XML, + JSON, + TEXT, + UNDEFINED +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java new file mode 100644 index 00000000..fa5930f1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNode.java @@ -0,0 +1,20 @@ +package eu.dnetlib.manager.wf.annotations; + +public @interface WfNode { + + String name(); + + WfNodeOperation operation(); + + Class inputStreamType() default void.class; + + StreamMimeType inputStreamMimeType() default StreamMimeType.UNDEFINED; + + Class outputStreamType() default void.class; + + StreamMimeType outputStreamMimeType() default StreamMimeType.UNDEFINED; + + WfParam[] inputParams() default {}; + + WfParam[] outputParams() default {}; +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java new file mode 100644 index 00000000..05a88378 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfNodeOperation.java @@ -0,0 +1,11 @@ +package eu.dnetlib.manager.wf.annotations; + +public enum WfNodeOperation { + CREATE, + DROP, + READ, + WRITE, + PRODUCER, + TRANSFORM, + SETENV +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java new file mode 100644 index 00000000..5566dc12 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/annotations/WfParam.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.annotations; + +public @interface WfParam { + + String name(); + + String type(); +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java new file mode 100644 index 00000000..2472c8a0 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/cron/ScheduledWorkflowLauncher.java @@ -0,0 +1,138 @@ +package eu.dnetlib.manager.wf.cron; + +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.support.CronExpression; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.wf.WorkflowManagerService; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository; +import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; + +@Service +public class ScheduledWorkflowLauncher { + + private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class); + + @Autowired + private WorkflowManagerService wfManagerService; + + @Autowired + private ProcessRegistry processRegistry; + + @Autowired + private WorkflowInstanceRepository workflowInstanceRepository; + + @Autowired + private WorkflowLogger logger; + + @Value("${dnet.workflow.scheduler.windowSize:1800000}") + private int windowSize; // 1800000 are 30 minutes + + @Scheduled(fixedRateString = "${dnet.workflow.scheduler.fixedRate:900000}") // 900000 are 5 minutes + public void verifySheduledWorkflows() { + log.debug("Verifying scheduled workflows - START"); + + workflowInstanceRepository.findAll() + .stream() + .filter(WorkflowInstance::isEnabled) + .filter(WorkflowInstance::isConfigured) + .filter(WorkflowInstance::isSchedulingEnabled) + .filter(this::isNotRunning) + .filter(this::isReady) + .forEach(instance -> { + try { + wfManagerService.startWorkflowInstance(instance, null); + } catch (final Exception e) { + log.error("Error launching scheduled wf instance: " + instance.getId(), e); + } + }); + log.debug("Verifying scheduled workflows - END"); + } + + private boolean isReady(final WorkflowInstance instance) { + final LocalDateTime lastExecutionDate = calculateLastExecutionDate(instance.getId()); + + final LocalDateTime now = LocalDateTime.now(); + + final String cron = instance.getCronExpression(); + + if (CronExpression.isValidExpression(cron)) { + final int minInterval = instance.getCronMinInterval(); // in minutes + + final boolean res; + if (lastExecutionDate != null) { + final long elapsed = ChronoUnit.MINUTES.between(lastExecutionDate, now); + res = elapsed > minInterval && verifyCron(cron, now); + } else { + res = verifyCron(cron, now); + } + + if (log.isDebugEnabled()) { + log.debug("**************************************************************"); + log.debug("WORKFLOW INSTANCE ID : " + instance.getId()); + log.debug("NOW : " + now); + log.debug("LAST EXECUTION DATE : " + lastExecutionDate); + log.debug("MIN INTERVAL (minutes) : " + minInterval); + log.debug("WINDOW SIZE (ms) : " + windowSize); + log.debug("MUST BE EXECUTED : " + res); + log.debug("**************************************************************"); + } + + return res; + } + + return false; + } + + private LocalDateTime calculateLastExecutionDate(final String id) { + return logger.getLastExecutionForInstance(id) + .map(e -> e.getEndDate()) + .orElse(LocalDateTime.MIN); + } + + private boolean verifyCron(final String cronExpression, final LocalDateTime now) { + try { + final CronExpression cron = CronExpression.parse(cronExpression); + + final LocalDateTime date = now.minus(windowSize, ChronoUnit.MINUTES); + final LocalDateTime nextDate = cron.next(date); + + if (log.isDebugEnabled()) { + log.debug("NEXT EXECUTION DATE: " + nextDate); + log.debug("FIRED : " + nextDate.isBefore(now)); + } + return nextDate.isBefore(now); + } catch (final Exception e) { + log.error("Error calculating next cron event: " + cronExpression, e); + return false; + } + } + + private boolean isNotRunning(final WorkflowInstance instance) { + final WorkflowProcess p = processRegistry.findProcsByInstanceId(instance.getId()); + + if (p != null) { + switch (p.getStatus()) { + case CREATED: + return false; + case EXECUTING: + return false; + default: + break; + } + } + + return true; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java new file mode 100644 index 00000000..21065752 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/AbstractJobNode.java @@ -0,0 +1,71 @@ +package eu.dnetlib.manager.wf.nodes; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; + +public abstract class AbstractJobNode extends ProcessNode { + + private static final Log log = LogFactory.getLog(AbstractJobNode.class); + + private final boolean async; + + public AbstractJobNode(final boolean async) { + this.async = async; + } + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + @Override + public final void execute(final Token token, final ExecutionCallback callback) { + try { + log.debug("START NODE: " + getBeanName()); + token.setProgressMessage(getNodeName()); + + beforeStart(token); + if (isAsync()) { + executor.execute(() -> doExecute(token, callback)); + } else { + doExecute(token, callback); + } + + log.debug("END NODE (SUCCESS): " + getBeanName()); + } catch (final Throwable e) { + log.error("got exception while executing workflow node", e); + log.debug("END NODE (FAILED): " + getBeanName()); + beforeFailed(token); + callback.onFail(token); + } + + } + + private final void doExecute(final Token token, final ExecutionCallback callback) { + execute(token); + beforeCompleted(token); + callback.onSuccess(token); + } + + protected abstract void execute(Token token); + + public final boolean isAsync() { + return async; + } + + protected void beforeStart(final Token token) { + // For optional overwrites + } + + protected void beforeCompleted(final Token token) { + // For optional overwrites + } + + protected void beforeFailed(final Token token) { + // For optional overwrites + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java new file mode 100644 index 00000000..b8321c00 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/CollectOAINode.java @@ -0,0 +1,29 @@ +package eu.dnetlib.manager.wf.nodes; + +import java.util.function.Supplier; +import java.util.stream.Stream; + +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import eu.dnetlib.manager.wf.annotations.StreamMimeType; +import eu.dnetlib.manager.wf.annotations.WfNode; +import eu.dnetlib.manager.wf.annotations.WfNodeOperation; +import eu.dnetlib.manager.wf.annotations.WfParam; + +@Component("oai_collect") +@Scope("prototype") +@WfNode(name = "oai_collect", operation = WfNodeOperation.PRODUCER, inputParams = { + @WfParam(name = "configuration", type = "ApiDescriptor.class"), +}, outputStreamType = String.class, outputStreamMimeType = StreamMimeType.XML) +public class CollectOAINode implements Supplier> { + + private String datasourceID; + + @Override + public Stream get() { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java new file mode 100644 index 00000000..ba6a54f8 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/DefaultJobNode.java @@ -0,0 +1,18 @@ +package eu.dnetlib.manager.wf.nodes; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 26/11/15. + */ +public final class DefaultJobNode extends AbstractJobNode { + + public DefaultJobNode(final String name) { + super(false); + setNodeName(name); + } + + @Override + public void execute(final Token token) {} + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java new file mode 100644 index 00000000..f05beb18 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/LaunchWorkflowJobNode.java @@ -0,0 +1,96 @@ +package eu.dnetlib.manager.wf.nodes; + +import java.util.HashMap; +import java.util.UUID; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import eu.dnetlib.manager.wf.WorkflowManagerService; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; +import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; + +/** + * Created by michele on 18/11/15. + */ +public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware { + + private static final Log log = LogFactory.getLog(LaunchWorkflowJobNode.class); + + private String wfId; + + @Autowired + private WorkflowManagerService wfManagerService; + + private WorkflowProcess process; + + @Override + public final void execute(final Token token, final ExecutionCallback callback) { + + try { + final WorkflowInstance instance = new WorkflowInstance(); + instance.setId("CHILD_" + UUID.randomUUID()); + instance.setParentId(process.getWfInstanceId()); + instance.setDetails(new HashMap<>()); + instance.setPriority(100); + instance.setDsId(process.getDsId()); + instance.setDsName(process.getDsName()); + instance.setApiId(process.getDsInterface()); + instance.setEnabled(true); + instance.setConfigured(true); + instance.setSchedulingEnabled(false); + instance.setCronExpression(""); + instance.setCronMinInterval(0); + instance.setWorkflow(wfId); + instance.setDestroyWf(null); + instance.setSystemParams(process.getGlobalParams()); + instance.setUserParams(new HashMap<>()); + + final String procId = wfManagerService.startWorkflowInstance(instance, new ExecutionCallback() { + + @Override + public void onSuccess(final WorkflowProcess t) { + log.debug("Child workflow has been completed successfully"); + token.release(); + callback.onSuccess(token); + } + + @Override + public void onFail(final WorkflowProcess t) { + log.error("Child workflow is failed"); + token.releaseAsFailed("Child workflow is failed"); + callback.onFail(token); + } + + }); + + if (log.isDebugEnabled()) { + log.debug("The child workflow [instance: " + instance.getId() + "] is starting with procId: " + procId); + } + + token.setProgressMessage("Launched sub workflow, proc: " + procId); + + } catch (final Throwable e) { + log.error("got exception while launching child workflow", e); + callback.onFail(token); + } + } + + @Override + public void setProcess(final WorkflowProcess process) { + this.process = process; + } + + public String getWfId() { + return wfId; + } + + public void setWfId(final String wfId) { + this.wfId = wfId; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java new file mode 100644 index 00000000..ea9aa2cb --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/NodeStatus.java @@ -0,0 +1,8 @@ +package eu.dnetlib.manager.wf.nodes; + +public enum NodeStatus { + CONFIGURED, + NOT_CONFIGURED, + DISABLED, + SYSTEM +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java new file mode 100644 index 00000000..f2d1c433 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/ProcessNode.java @@ -0,0 +1,38 @@ +package eu.dnetlib.manager.wf.nodes; + +import org.springframework.beans.factory.BeanNameAware; + +import eu.dnetlib.manager.wf.workflows.procs.Token; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; + +public abstract class ProcessNode implements BeanNameAware { + + private String beanName; + + private String nodeName; + + public abstract void execute(final Token token, ExecutionCallback callback); + + public String getBeanName() { + return this.beanName; + } + + @Override + public void setBeanName(final String beanName) { + this.beanName = beanName; + } + + public String getNodeName() { + return this.nodeName; + } + + public void setNodeName(final String nodeName) { + this.nodeName = nodeName; + } + + @Override + public String toString() { + return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java new file mode 100644 index 00000000..4339b5d1 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SimpleJobNode.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.nodes; + +public abstract class SimpleJobNode extends AbstractJobNode { + + public SimpleJobNode() { + super(false); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java new file mode 100644 index 00000000..c61c04be --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/SuccessNode.java @@ -0,0 +1,18 @@ +package eu.dnetlib.manager.wf.nodes; + +import eu.dnetlib.manager.wf.workflows.procs.Token; + +/** + * Created by michele on 26/11/15. + */ +public class SuccessNode extends AbstractJobNode { + + public SuccessNode() { + super(false); + setNodeName("success"); + } + + @Override + protected void execute(final Token token) {} + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java new file mode 100644 index 00000000..0f9d057a --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/notification/EmailSender.java @@ -0,0 +1,46 @@ +package eu.dnetlib.manager.wf.notification; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import eu.dnetlib.manager.wf.model.NotificationCondition; +import eu.dnetlib.manager.wf.repository.WorkflowSubscriptionRepository; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status; +import eu.dnetlib.notifications.mail.EmailService; + +@Service +public class EmailSender { + + private static final Log log = LogFactory.getLog(EmailSender.class); + + @Autowired + private WorkflowSubscriptionRepository wfSubscriptionRepository; + + @Autowired + private EmailService emailService; + + public void sendMails(final WorkflowProcess proc) { + + wfSubscriptionRepository.findByWfInstanceId(proc.getWfInstanceId()).forEach(s -> { + if (s.getCondition() == NotificationCondition.ALWAYS || + s.getCondition() == NotificationCondition.ONLY_FAILED && proc.getStatus() == Status.FAILURE || + s.getCondition() == NotificationCondition.ONLY_SUCCESS && proc.getStatus() == Status.SUCCESS) { + try { + final Map params = new HashMap<>(); + + emailService.sendStoredMail(s.getEmail(), s.getMessageId(), params); + + } catch (final Exception e) { + log.error("Error sending mail to " + s.getEmail(), e); + } + } + + }); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java new file mode 100644 index 00000000..1d60254b --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Arc.java @@ -0,0 +1,40 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import com.google.common.base.Function; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class Arc { + + private final String from; + private final String to; + private final Function condFunction; + + public Arc(final String from, final String to, final Function condFunction) { + this.from = from; + this.to = to; + this.condFunction = condFunction; + } + + public String getFrom() { + return this.from; + } + + public String getTo() { + return this.to; + } + + public boolean isValid(final Env env) { + if (condFunction != null) { + return condFunction.apply(env); + } else { + return true; + } + } + + @Override + public String toString() { + return String.format("[ %s -> %s, %s ]", this.from, this.to, this.condFunction != null ? "with cond" : "without cond"); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java new file mode 100644 index 00000000..a09b5f91 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/Graph.java @@ -0,0 +1,84 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class Graph { + + private final Map nodes = new HashMap<>(); + private List arcs = new ArrayList<>(); + + public void addArc(final Arc arc) { + this.arcs.add(arc); + } + + public void addNode(final GraphNode node) { + this.nodes.put(node.getName(), node); + } + + public Set nodeNames() { + return this.nodes.keySet(); + } + + public Collection nodes() { + return this.nodes.values(); + } + + public GraphNode getNode(final String name) { + return this.nodes.get(name); + } + + public List getArcs() { + return this.arcs; + } + + public void setArcs(final List arcs) { + this.arcs = arcs; + } + + public Set startNodes() { + final Set res = new HashSet<>(); + for (final GraphNode n : this.nodes.values()) { + if (n.isStart()) { + res.add(n); + } + } + return res; + } + + public Set nextNodes(final GraphNode current, final Env env) { + return arcs.stream() + .filter(arc -> StringUtils.equals(arc.getFrom(), current.getName())) + .filter(arc -> arc.isValid(env)) + .map(arc -> arc.getTo()) + .distinct() + .map(to -> nodes.get(to)) + .collect(Collectors.toSet()); + + } + + public int getNumberOfIncomingArcs(final GraphNode node) { + int count = 0; + for (final Arc arc : this.arcs) { + if (arc.getTo().equals(node.getName())) { + count++; + } + } + return count; + } + + @Override + public String toString() { + return "\n************************\nNodes: " + this.nodes + "\nArcs: " + this.arcs + "\n************************\n"; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java new file mode 100644 index 00000000..ff9cfa29 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoader.java @@ -0,0 +1,114 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.expression.MapAccessor; +import org.springframework.core.env.Environment; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.stereotype.Service; + +import com.google.common.base.Function; +import com.google.common.collect.Sets; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.WorkflowGraph; +import eu.dnetlib.manager.wf.workflows.procs.Env; +import eu.dnetlib.manager.wf.workflows.util.NodeHelper; + +@Service +public class GraphLoader { + + @Autowired + private NodeHelper nodeHelper; + + @Autowired + private Environment environment; + + public Graph loadGraph(final WorkflowGraph workflowGraph, final Map globalParams) + throws WorkflowManagerException { + final Graph graph = new Graph(); + + for (final WorkflowGraph.Node node : workflowGraph.getGraph()) { + final String nodeName = node.getName(); + final String nodeType = node.getType(); + final boolean isStart = node.isStart(); + final boolean isJoin = node.isJoin(); + + final Map params = node.calculateInitialParams(globalParams, environment); + final Map envParams = node.findEnvParams(); + + if (isStart) { + graph.addNode(GraphNode.newStartNode(nodeName, nodeType, params, envParams)); + } else if (isJoin) { + graph.addNode(GraphNode.newJoinNode(nodeName, nodeType, params, envParams)); + } else { + graph.addNode(GraphNode.newNode(nodeName, nodeType, params, envParams)); + } + + if (node.getArcs() != null) { + for (final WorkflowGraph.Arc a : node.getArcs()) { + final String to = a.getTo(); + final Function condFunction = generateFunction(a.getCondition()); + graph.addArc(new Arc(nodeName, to, condFunction)); + } + } + + graph.addNode(GraphNode.newSuccessNode()); + } + + checkValidity(graph); + + return graph; + } + + private Function generateFunction(final String condition) { + return env -> { + final ExpressionParser parser = new SpelExpressionParser(); + + final StandardEvaluationContext context = new StandardEvaluationContext(env.getAttributes()); + context.addPropertyAccessor(new MapAccessor()); + + return parser.parseExpression(condition).getValue(context, Boolean.class); + }; + } + + private void checkValidity(final Graph graph) throws WorkflowManagerException { + + final Set nodesFromArcs = new HashSet<>(); + + boolean foundSuccess = false; + boolean foundStart = false; + + for (final Arc arc : graph.getArcs()) { + if (StringUtils.isBlank(arc.getFrom()) || StringUtils.isBlank(arc.getFrom())) { + throw new WorkflowManagerException("Invalid arc: missing from e/o to"); + } + if (StringUtils.equals(arc.getTo(), GraphNode.SUCCESS_NODE)) { + foundSuccess = true; + } + nodesFromArcs.add(arc.getFrom()); + nodesFromArcs.add(arc.getTo()); + } + + if (!foundSuccess) { throw new WorkflowManagerException("Arc to success not found"); } + + final Set diff = Sets.symmetricDifference(graph.nodeNames(), nodesFromArcs); + if (!diff.isEmpty()) { throw new WorkflowManagerException("Missing or invalid nodes in arcs: " + diff); } + + for (final GraphNode n : graph.nodes()) { + if (StringUtils.isBlank(n.getName())) { throw new WorkflowManagerException("Invalid node: missing name"); } + if (n.isStart()) { + foundStart = true; + } + if (!this.nodeHelper.isValidType(n.getType())) { throw new WorkflowManagerException("Invalid node type: " + n.getType()); } + } + if (!foundStart) { throw new WorkflowManagerException("Start node not found"); } + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java new file mode 100644 index 00000000..9f3429e2 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/graph/GraphNode.java @@ -0,0 +1,129 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class GraphNode { + + public static final String SUCCESS_NODE = "success"; + + private final String name; + private final String type; + private final boolean isStart; + private final boolean isJoin; + private final boolean isSucessNode; + private final Map params; + private final Map envParams; + + private GraphNode(final String name, + final String type, + final boolean isStart, + final boolean isJoin, + final boolean isSuccessNode, + final Map params, + final Map envParams) { + this.name = name; + this.type = type; + this.isStart = isStart; + this.isJoin = isJoin; + this.isSucessNode = isSuccessNode; + this.params = params; + this.envParams = envParams; + } + + public static GraphNode newNode(final String name, + final String type, + final Map params, + final Map envParams) { + return new GraphNode(name, type, false, false, false, params, envParams); + } + + public static GraphNode newStartNode(final String name, + final String type, + final Map params, + final Map envParams) { + return new GraphNode(name, type, true, false, false, params, envParams); + } + + public static GraphNode newJoinNode(final String name, + final String type, + final Map params, + final Map envParams) { + return new GraphNode(name, type, false, true, false, params, envParams); + } + + public static GraphNode newSuccessNode() { + return new GraphNode(SUCCESS_NODE, null, false, true, true, new HashMap<>(), new HashMap<>()); + } + + public String getName() { + return this.name; + } + + public String getType() { + return this.type; + } + + public boolean isStart() { + return this.isStart; + } + + public boolean isJoin() { + return this.isJoin; + } + + public boolean isSucessNode() { + return this.isSucessNode; + } + + @Override + public String toString() { + final StringWriter sw = new StringWriter(); + sw.append("[ name: "); + sw.append(this.name); + if (StringUtils.isNotBlank(this.type)) { + sw.append(", type: "); + sw.append(this.type); + } + if (isStart()) { + sw.append(" - isStart"); + } + if (isJoin()) { + sw.append(" - isJoin"); + } + sw.append(" ]"); + return sw.toString(); + } + + public Map getParams() { + return this.params; + } + + public Map getEnvParams() { + return this.envParams; + } + + public Map resolveParamsWithNoEnv() { + return resolveParams(null); + } + + public Map resolveParams(final Env env) { + final Map map = new HashMap<>(); + + if (this.params != null) { + this.params.forEach((k, v) -> map.put(k, v)); + } + + if (this.envParams != null && env != null) { + this.envParams.forEach((k, v) -> map.put(k, env.getAttribute(v))); + } + + return map; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java new file mode 100644 index 00000000..30ef34e7 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Env.java @@ -0,0 +1,54 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by michele on 23/11/15. + */ +public class Env { + + private final Map attrs; + + public Env() { + this.attrs = new HashMap<>(); + } + + public Env(final Map attrs) { + this.attrs = attrs; + } + + public Map getAttributes() { + return attrs; + } + + public void clear() { + attrs.clear(); + } + + public void addAttributes(final Map map) { + if (map != null) { + attrs.putAll(map); + } + } + + public void setAttribute(final String name, final Object value) { + attrs.put(name, value); + } + + public Object getAttribute(final String name) { + return attrs.get(name); + } + + public T getAttribute(final String name, Class clazz) { + return clazz.cast(attrs.get(name)); + } + + public boolean hasAttribute(final String name) { + return attrs.containsKey(name); + } + + public Object removeAttribute(final String name) { + return attrs.remove(name); + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java new file mode 100644 index 00000000..37645ccb --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessAware.java @@ -0,0 +1,10 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +/** + * Created by michele on 24/11/15. + */ +public interface ProcessAware { + + void setProcess(WorkflowProcess process); + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java new file mode 100644 index 00000000..6585c39b --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessEngine.java @@ -0,0 +1,137 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.common.base.Throwables; + +import eu.dnetlib.manager.history.WorkflowLogger; +import eu.dnetlib.manager.wf.nodes.ProcessNode; +import eu.dnetlib.manager.wf.notification.EmailSender; +import eu.dnetlib.manager.wf.workflows.graph.GraphNode; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; +import eu.dnetlib.manager.wf.workflows.util.NodeHelper; + +@Service +public class ProcessEngine { + + private static final Log log = LogFactory.getLog(ProcessEngine.class); + + @Autowired + private NodeHelper nodeHelper; + @Autowired + private EmailSender emailSender; + @Autowired + private WorkflowLogger wfLogger; + + public void startProcess(final WorkflowProcess process) { + log.info(process.getGraph()); + + log.info("Starting workflow: " + process); + + final LocalDateTime now = LocalDateTime.now(); + process.setStatus(WorkflowProcess.Status.EXECUTING); + process.setStartDate(now); + process.setLastActivityDate(now); + + try { + for (final GraphNode node : process.getGraph().startNodes()) { + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv()); + final Token token = new Token(); + token.getEnv().addAttributes(process.getEnv().getAttributes()); + process.getTokens().add(token); + + pNode.execute(token, newNodeCallback(process, node)); + } + } catch (final Throwable e) { + log.error("WorkflowProcess node instantiation failed", e); + process.setStatus(WorkflowProcess.Status.FAILURE); + } + } + + private ExecutionCallback newNodeCallback(final WorkflowProcess process, final GraphNode node) { + return new ExecutionCallback() { + + @Override + public void onSuccess(final Token t) { + releaseToken(process, node, t); + } + + @Override + public void onFail(final Token t) { + completeProcess(process, t); + } + }; + } + + public void releaseToken(final WorkflowProcess process, final GraphNode oldGraphNode, final Token oldToken) { + process.setLastActivityDate(LocalDateTime.now()); + + try { + for (final GraphNode node : process.getGraph().nextNodes(oldGraphNode, oldToken.getEnv())) { + if (node.isJoin() || node.isSucessNode()) { + if (!process.getPausedJoinNodeTokens().containsKey(node.getName())) { + process.getPausedJoinNodeTokens().put(node.getName(), new ArrayList()); + } + + final List list = process.getPausedJoinNodeTokens().get(node.getName()); + + list.add(oldToken); + + if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) { + final Token token = new Token(); + token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()]))); + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); + + process.getTokens().add(token); + process.setLastActivityDate(LocalDateTime.now()); + + if (node.isSucessNode()) { + completeProcess(process, token); + } else { + pNode.execute(token, newNodeCallback(process, node)); + } + } + } else { + final Token token = new Token(); + token.getEnv().addAttributes(oldToken.getEnv().getAttributes()); + final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv()); + + process.getTokens().add(token); + process.setLastActivityDate(LocalDateTime.now()); + pNode.execute(token, newNodeCallback(process, node)); + } + } + } catch (final Throwable e) { + log.error("WorkflowProcess node instantiation failed", e); + process.setStatus(WorkflowProcess.Status.FAILURE); + process.setError(e.getMessage()); + process.setErrorStacktrace(Throwables.getStackTraceAsString(e)); + process.setLastActivityDate(LocalDateTime.now()); + } + + } + + private Map mergeEnvParams(final Token... tokens) { + final Map map = new HashMap<>(); + Arrays.stream(tokens).forEach(t -> map.putAll(t.getEnv().getAttributes())); + return map; + } + + private void completeProcess(final WorkflowProcess process, final Token token) { + token.checkStatus(); + process.complete(token); + wfLogger.saveProcessExecution(process.asLog()); + emailSender.sendMails(process); + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java new file mode 100644 index 00000000..68c493bf --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessFactory.java @@ -0,0 +1,59 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.manager.wf.model.WorkflowGraph; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.graph.Graph; +import eu.dnetlib.manager.wf.workflows.graph.GraphLoader; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; + +@Component +public class ProcessFactory { + + private static final Log log = LogFactory.getLog(ProcessFactory.class); + + private String oldGeneratedId = ""; + + private final DateTimeFormatter processIdFormatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_S"); + + @Autowired + private GraphLoader graphLoader; + + public WorkflowProcess newProcess(final SimpleResource wfMetadata, + final WorkflowGraph wfGraph, + final WorkflowInstance wfInstance, + final ExecutionCallback callback) throws WorkflowManagerException { + + final Map globalParams = new HashMap<>(); + globalParams.putAll(wfInstance.getSystemParams()); + globalParams.putAll(wfInstance.getUserParams()); + + final Graph graph = graphLoader.loadGraph(wfGraph, globalParams); + + return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, callback); + + } + + private synchronized String generateProcessId() { + String id = ""; + do { + id = "wf_" + LocalDateTime.now().format(processIdFormatter); + log.info("Generated processID " + id); + } while (id.equals(oldGeneratedId)); + + oldGeneratedId = id; + + return id; + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java new file mode 100644 index 00000000..c9e0dbf2 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/ProcessRegistry.java @@ -0,0 +1,124 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.PriorityBlockingQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; + +@Service +public class ProcessRegistry { + + private static final Log log = LogFactory.getLog(ProcessRegistry.class); + private final Map procs = new HashMap<>(); + private final Map byInstanceId = new HashMap<>(); + + private final PriorityBlockingQueue pendingProcs = new PriorityBlockingQueue<>(); + + @Value("${dnet.wf.registry.size:100}") + private int maxSize; + + synchronized public int countRunningWfs() { + int count = 0; + for (final Map.Entry e : this.procs.entrySet()) { + final WorkflowProcess proc = e.getValue(); + if (!proc.isTerminated()) { + count++; + } + } + return count; + } + + public WorkflowProcess findProcess(final String procId) { + return this.procs.get(procId); + } + + public Collection listProcesses() { + return this.procs.values(); + } + + public WorkflowProcess findProcsByInstanceId(final String id) { + return this.byInstanceId.get(id); + } + + public String registerProcess(final WorkflowProcess process, final WorkflowInstance wfInstance) throws WorkflowManagerException { + if (this.procs.containsValue(process) || this.procs.containsKey(process.getId())) { + log.error("Already registerd process: " + process); + throw new WorkflowManagerException("Already registerd process: " + process); + } + + if (this.procs.size() >= this.maxSize) { + removeOldestProcess(); + } + + this.procs.put(process.getId(), process); + this.byInstanceId.put(wfInstance.getId(), process); + + synchronized (this.pendingProcs) { + if (this.pendingProcs.size() > WorkflowsConstants.MAX_PENDING_PROCS_SIZE) { + log.warn("Wf [" + process.getName() + "] not launched, Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE); + throw new WorkflowManagerException("Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE); + } + this.pendingProcs.put(process); + + log.info("WorkflowProcess [" + process + "] in queue, priority=" + process.getPriority()); + } + + return process.getId(); + } + + private void removeOldestProcess() { + LocalDateTime oldDate = LocalDateTime.now(); + String oldId = null; + + for (final Map.Entry e : this.procs.entrySet()) { + final WorkflowProcess proc = e.getValue(); + + if (proc.isTerminated()) { + final LocalDateTime date = proc.getLastActivityDate(); + if (date.isBefore(oldDate)) { + oldDate = date; + oldId = e.getKey(); + } + } + } + + if (oldId != null) { + unregisterProcess(oldId); + } + + } + + public void unregisterProcess(final String procId) { + synchronized (this) { + final WorkflowProcess process = this.procs.remove(procId); + if (process != null) { + final Optional instanceId = this.byInstanceId.entrySet() + .stream() + .filter(e -> e.getValue().getId().equals(process.getId())) + .map(e -> e.getKey()) + .findFirst(); + if (instanceId.isPresent()) { + this.byInstanceId.remove(instanceId, process); + } + } + } + } + + public WorkflowProcess nextProcessToStart() { + synchronized (this.pendingProcs) { + return this.pendingProcs.poll(); + } + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java new file mode 100644 index 00000000..bfb1541e --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/Token.java @@ -0,0 +1,125 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.UUID; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Throwables; + +/** + * Created by michele on 19/11/15. + */ +public class Token { + + private final String id; + private final Env env = new Env(); + + private String progressMessage; + private final LocalDateTime startDate; + private LocalDateTime endDate; + + private boolean failed; + private boolean active; + + private String error = ""; + private String errorStackTrace = ""; + + public Token() { + this.id = "token-" + UUID.randomUUID(); + this.startDate = LocalDateTime.now(); + this.failed = false; + this.active = true; + } + + public String getId() { + return this.id; + } + + public Env getEnv() { + return this.env; + } + + public LocalDateTime getStartDate() { + return this.startDate; + } + + public LocalDateTime getEndDate() { + return this.endDate; + } + + public void setEndDate(final LocalDateTime endDate) { + this.endDate = endDate; + } + + public boolean isActive() { + return this.active; + } + + public void setActive(final boolean active) { + this.active = active; + } + + public boolean isFailed() { + return this.failed; + } + + public void setFailed(final boolean failed) { + this.failed = failed; + } + + public void release() { + setEndDate(LocalDateTime.now()); + setActive(false); + } + + public void releaseAsFailed(final Throwable e) { + setEndDate(LocalDateTime.now()); + setActive(false); + setFailed(true); + setError(e.getMessage()); + setErrorStackTrace(Throwables.getStackTraceAsString(e)); + } + + public void releaseAsFailed(final String error) { + setEndDate(LocalDateTime.now()); + setActive(false); + setFailed(true); + setError(error); + } + + public void checkStatus() { + if (isActive()) { + if (StringUtils.isNotBlank(error)) { + releaseAsFailed(error); + } else { + release(); + } + } + } + + public String getProgressMessage() { + return progressMessage; + } + + public void setProgressMessage(final String progressMessage) { + this.progressMessage = progressMessage; + } + + public String getError() { + return this.error; + } + + public void setError(final String error) { + this.error = error; + } + + public String getErrorStackTrace() { + return this.errorStackTrace; + } + + public void setErrorStackTrace(final String errorStackTrace) { + this.errorStackTrace = errorStackTrace; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java new file mode 100644 index 00000000..caf32529 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/procs/WorkflowProcess.java @@ -0,0 +1,263 @@ +package eu.dnetlib.manager.wf.workflows.procs; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.lang3.math.NumberUtils; + +import eu.dnetlib.is.model.resource.SimpleResource; +import eu.dnetlib.manager.history.model.WfProcessExecution; +import eu.dnetlib.manager.wf.model.WorkflowInstance; +import eu.dnetlib.manager.wf.workflows.graph.Graph; +import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback; +import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants; + +/** + * Created by michele on 19/11/15. + */ +public class WorkflowProcess implements Comparable { + + public enum Status { + CREATED, + EXECUTING, + SUCCESS, + FAILURE, + KILLED; + } + + public enum StartMode { + AUTO, + MANUAL, + DISABLED + } + + private final String id; + private final SimpleResource wfMetadata; + private final WorkflowInstance wfInstance; + private final Graph graph; + private final ExecutionCallback callback; + private final Env env; + private final List tokens = new CopyOnWriteArrayList<>(); + private LocalDateTime lastActivityDate; + private Status status; + private LocalDateTime startDate = LocalDateTime.MIN; + private LocalDateTime endDate = LocalDateTime.MIN; + private final Map> pausedJoinNodeTokens = new HashMap<>(); + private final Map globalParams; + private String error; + private String errorStacktrace; + private final Map outputParams = new HashMap<>(); + + public WorkflowProcess( + final String id, + final SimpleResource wfMetadata, + final WorkflowInstance wfInstance, + final Graph graph, + final Map globalParams, + final ExecutionCallback callback) { + this.id = id; + this.wfMetadata = wfMetadata; + this.wfInstance = wfInstance; + this.graph = graph; + this.callback = callback; + this.status = Status.CREATED; + this.env = new Env(); + this.globalParams = globalParams; + this.lastActivityDate = LocalDateTime.now(); + } + + public String getId() { + return id; + } + + public String getName() { + return wfInstance.getName(); + } + + public String getFamily() { + return wfMetadata.getSubtype(); + } + + public String getWfId() { + return wfMetadata.getId(); + } + + public String getWfInstanceId() { + return wfInstance.getId(); + } + + public String getParentId() { + return wfInstance.getParentId(); + } + + public int getPriority() { + return wfInstance.getPriority(); + } + + public String getDsId() { + return wfInstance.getId(); + } + + public String getDsName() { + return wfInstance.getDsName(); + } + + public String getDsInterface() { + return wfInstance.getApiId(); + } + + public Map> getPausedJoinNodeTokens() { + return pausedJoinNodeTokens; + } + + public Env getEnv() { + return env; + } + + public Status getStatus() { + return status; + } + + public void setStatus(final Status status) { + this.status = status; + } + + public Graph getGraph() { + return graph; + } + + public List getTokens() { + return tokens; + } + + public void kill() { + setStatus(Status.KILLED); + } + + public boolean isTerminated() { + switch (status) { + case SUCCESS: + case FAILURE: + case KILLED: + return true; + default: + return false; + } + } + + public LocalDateTime getLastActivityDate() { + return lastActivityDate; + } + + public void setLastActivityDate(final LocalDateTime lastActivityDate) { + this.lastActivityDate = lastActivityDate; + } + + @Override + public String toString() { + return String.format("[process id='%s' name='%s']", id, wfMetadata.getName()); + } + + @Override + public int compareTo(final WorkflowProcess wp) { + return NumberUtils.compare(getPriority(), wp.getPriority()); + } + + public Map getGlobalParams() { + return globalParams; + } + + public void setStartDate(final LocalDateTime startDate) { + this.startDate = startDate; + } + + public void setEndDate(final LocalDateTime endDate) { + this.endDate = endDate; + } + + public LocalDateTime getStartDate() { + return startDate; + } + + public LocalDateTime getEndDate() { + return endDate; + } + + public String getError() { + return error; + } + + public void setError(final String error) { + this.error = error; + } + + public String getErrorStacktrace() { + return errorStacktrace; + } + + public void setErrorStacktrace(final String errorStacktrace) { + this.errorStacktrace = errorStacktrace; + } + + public Map getOutputParams() { + return outputParams; + } + + public void complete(final Token token) { + final LocalDateTime now = token.getEndDate(); + setLastActivityDate(now); + setEndDate(now); + setStatus(token.isFailed() ? WorkflowProcess.Status.FAILURE : WorkflowProcess.Status.SUCCESS); + + if (token.isFailed()) { + setStatus(Status.FAILURE); + setError(token.getError()); + setErrorStacktrace(token.getErrorStackTrace()); + setLastActivityDate(LocalDateTime.now()); + } + + if (callback != null) { + if (token.isFailed()) { + callback.onFail(this); + } else { + callback.onSuccess(this);; + } + } + + } + + public WfProcessExecution asLog() { + final Map details = new LinkedHashMap<>(); + details.putAll(getOutputParams()); + details.put(WorkflowsConstants.LOG_WF_PRIORITY, "" + getPriority()); + details.put(WorkflowsConstants.LOG_WF_ID, getWfId()); + details.put(WorkflowsConstants.LOG_WF_ID, getWfInstanceId()); + + if (getError() != null) { + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR, getError()); + details.put(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, getErrorStacktrace()); + } + + final WfProcessExecution pe = new WfProcessExecution(); + pe.setProcessId(getId()); + pe.setName(getName()); + pe.setFamily(getFamily()); + + pe.setDsId(getDsId()); + pe.setDsName(getDsName()); + pe.setDsApi(getDsInterface()); + + pe.setStartDate(getStartDate()); + pe.setEndDate(getEndDate()); + pe.setStatus(getStatus().toString()); + + pe.setDetails(details); + + return pe; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java new file mode 100644 index 00000000..04bcdf38 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ExecutionCallback.java @@ -0,0 +1,9 @@ +package eu.dnetlib.manager.wf.workflows.util; + +public interface ExecutionCallback { + + void onSuccess(T t); + + void onFail(T t); + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java new file mode 100644 index 00000000..c9c11307 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/NodeHelper.java @@ -0,0 +1,65 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.PropertyAccessorFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.nodes.ProcessNode; +import eu.dnetlib.manager.wf.nodes.DefaultJobNode; +import eu.dnetlib.manager.wf.nodes.SuccessNode; +import eu.dnetlib.manager.wf.workflows.graph.GraphNode; +import eu.dnetlib.manager.wf.workflows.procs.Env; +import eu.dnetlib.manager.wf.workflows.procs.ProcessAware; +import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess; + +@Component +public class NodeHelper implements ApplicationContextAware { + + public static final String beanNamePrefix = "wfNode"; + private static final Log log = LogFactory.getLog(NodeHelper.class); + + private ApplicationContext applicationContext; + + public ProcessNode newProcessNode(final GraphNode node, final WorkflowProcess process, final Env env) throws WorkflowManagerException { + if (node.isSucessNode()) { + return new SuccessNode(); + } else if (StringUtils.isBlank(node.getType())) { + return new DefaultJobNode(node.getName()); + } else { + final ProcessNode pnode = this.applicationContext.getBean(beanNamePrefix + node.getType(), ProcessNode.class); + if (pnode != null) { + pnode.setNodeName(node.getName()); + // I invoke the setter methods using the static params of the graph node + try { + PropertyAccessorFactory.forBeanPropertyAccess(pnode).setPropertyValues(node.resolveParams(env)); + } catch (final Throwable e) { + throw new WorkflowManagerException(String.format("error setting parameters in wfNode %s", node.getName()), e); + } + if (pnode instanceof ProcessAware) { + ((ProcessAware) pnode).setProcess(process); + } + return pnode; + } else { + log.error("cannot find bean " + beanNamePrefix + node.getType()); + throw new WorkflowManagerException("cannot find bean " + beanNamePrefix + node.getType()); + } + } + } + + public boolean isValidType(final String type) { + return StringUtils.isBlank(type) || this.applicationContext.isPrototype(beanNamePrefix + type) && this.applicationContext + .isTypeMatch(beanNamePrefix + type, ProcessNode.class); + } + + @Override + public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java new file mode 100644 index 00000000..956abac4 --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/ValidNodeValuesFetcher.java @@ -0,0 +1,71 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Required; + +import eu.dnetlib.errors.WorkflowManagerException; +import eu.dnetlib.manager.wf.workflows.util.ValidNodeValuesFetcher.DnetParamValue; + +public abstract class ValidNodeValuesFetcher implements Function, List> { + + private String name; + + private static final Log log = LogFactory.getLog(ValidNodeValuesFetcher.class); + + public class DnetParamValue implements Comparable { + + private final String id; + private final String name; + + public DnetParamValue(final String id, final String name) { + this.id = id; + this.name = name; + } + + public String getId() { + return this.id; + } + + public String getName() { + return this.name; + } + + @Override + public int compareTo(final DnetParamValue o) { + return getName().compareTo(o.getName()); + } + } + + @Override + final public List apply(final Map params) { + try { + return obtainValues(params); + } catch (final Throwable e) { + log.error("Error obtaing values", e); + return new ArrayList<>(); + } + } + + abstract protected List obtainValues(Map params) throws Exception; + + public String getName() { + return this.name; + } + + @Required + public void setName(final String name) { + this.name = name; + } + + protected void verifyParams(final Map params, final String... pnames) throws WorkflowManagerException { + for (final String s : pnames) { + if (!params.containsKey(s)) { throw new WorkflowManagerException("Parameter not found: " + s); } + } + } +} diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java new file mode 100644 index 00000000..3be95afc --- /dev/null +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/workflows/util/WorkflowsConstants.java @@ -0,0 +1,119 @@ +package eu.dnetlib.manager.wf.workflows.util; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Required; + +import com.google.gson.Gson; + +public class WorkflowsConstants { + + public static final String DATASOURCE_PREFIX = "datasource:"; + + public static final String LOG_WF_NAME = "system:wfName"; + public static final String LOG_WF_ID = "system:wfId"; + public static final String LOG_WF_INSTANCE_ID = "system:wfInstanceId"; + + public static final String LOG_WF_FAMILY = "system:family"; + public static final String LOG_WF_PRIORITY = "system:priority"; + public static final String LOG_WF_PROCESS_ID = "system:processId"; + public static final String LOG_WF_PROCESS_STATUS = "system:processStatus"; + public static final String LOG_WF_PROCESS_START_DATE = "system:startDate"; + public static final String LOG_WF_PROCESS_END_DATE = "system:endDate"; + public static final String LOG_WF_PARENT = "system:parentProfileId"; + + public static final String LOG_SYSTEM_ERROR = "system:error"; + public static final String LOG_SYSTEM_ERROR_STACKTRACE = "system:error:stacktrace"; + + public static final String LOG_DATASOURCE_NAME = WorkflowsConstants.DATASOURCE_PREFIX + "name"; + public static final String LOG_DATASOURCE_ID = WorkflowsConstants.DATASOURCE_PREFIX + "id"; + public static final String LOG_DATASOURCE_INTERFACE = WorkflowsConstants.DATASOURCE_PREFIX + "interface"; + + public static final String BLACKBOARD_IS_BLACKBOARD = "blackboard:isBlackboard"; + public static final String BLACKBOARD_JOB = "blackboard:job"; + public static final String BLACKBOARD_SERVICE_ID = "blackboard:serviceId"; + public static final String BLACKBOARD_IS_GOING = "blackboard:isOngoing"; + public static final String BLACKBOARD_PARAM_PREFIX = "blackboard:param:"; + + // public static final String DATASOURCE_ACRONYM = WorkflowsConstants.DATASOURCE_PREFIX + "acronym"; + // public static final String DATASOURCE_URL = WorkflowsConstants.DATASOURCE_PREFIX + "url"; + + public static final int MIN_WF_PRIORITY = 0; + public static final int MAX_WF_PRIORITY = 100; + public static final int DEFAULT_WF_PRIORITY = 50; + public static final int MAX_PENDING_PROCS_SIZE = 100; + public static final int MAX_RUNNING_PROCS_SIZE = 100; + + public static final String MAIN_LOG_PREFIX = "mainlog:"; + + private String datasourceProtocolsJson; + private String datasourceTypologiesJson; + private List> datasourceProtocols; + private List> datasourceTypologies; + private List> datasourceWorkflowStatuses; + + @SuppressWarnings("unchecked") + public void init() { + final Gson gson = new Gson(); + datasourceProtocols = gson.fromJson(datasourceProtocolsJson, List.class); + datasourceTypologies = gson.fromJson(datasourceTypologiesJson, List.class); + datasourceWorkflowStatuses = new ArrayList<>(); + for (final WorkflowStatus s : WorkflowStatus.values()) { + final Map map = new HashMap<>(); + map.put("name", s.displayName); + map.put("icon", s.icon); + map.put("value", s.toString()); + datasourceWorkflowStatuses.add(map); + } + } + + public String getDatasourceProtocolsJson() { + return datasourceProtocolsJson; + } + + @Required + public void setDatasourceProtocolsJson(final String datasourceProtocolsJson) { + this.datasourceProtocolsJson = datasourceProtocolsJson; + } + + public String getDatasourceTypologiesJson() { + return datasourceTypologiesJson; + } + + @Required + public void setDatasourceTypologiesJson(final String datasourceTypologiesJson) { + this.datasourceTypologiesJson = datasourceTypologiesJson; + } + + public List> getDatasourceProtocols() { + return datasourceProtocols; + } + + public List> getDatasourceTypologies() { + return datasourceTypologies; + } + + public List> getDatasourceWorkflowStatuses() { + return datasourceWorkflowStatuses; + } + + public enum WorkflowStatus { + + EXECUTABLE("Executable", "icon-ok"), + WAIT_USER_SETTINGS("Waiting user settings", "icon-edit"), + WAIT_SYS_SETTINGS("Waiting system settings", + "icon-refresh"); + + public String displayName; + public String icon; + + WorkflowStatus(final String displayName, final String icon) { + this.displayName = displayName; + this.icon = icon; + } + } + +} diff --git a/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java b/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java new file mode 100644 index 00000000..f76b610b --- /dev/null +++ b/libs/dnet-wf-service/src/test/java/eu/dnetlib/manager/wf/workflows/graph/GraphLoaderTest.java @@ -0,0 +1,54 @@ +package eu.dnetlib.manager.wf.workflows.graph; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.function.Function; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.expression.MapAccessor; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import eu.dnetlib.manager.wf.workflows.procs.Env; + +public class GraphLoaderTest { + + private Env env; + + @BeforeEach + void setUp() throws Exception { + env = new Env(); + env.setAttribute("author", "Michele Artini"); + env.setAttribute("age", 47); + } + + @Test + final void testExpressions() { + assertTrue(evalFunction("age == 47").apply(env)); + assertTrue(evalFunction("age > 40").apply(env)); + assertTrue(evalFunction("author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 && author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 || author == 'Michele Artini'").apply(env)); + assertTrue(evalFunction("age == 47 || author == 'Claudio Atzori'").apply(env)); + assertTrue(evalFunction("age == 22 || author == 'Michele Artini'").apply(env)); + assertFalse(evalFunction("age != 47").apply(env)); + assertFalse(evalFunction("age < 40").apply(env)); + assertFalse(evalFunction("author != 'Michele Artini'").apply(env)); + assertFalse(evalFunction("age == 47 && author == 'Claudio Atzori'").apply(env)); + } + + private Function evalFunction(final String f) { + return env -> { + final ExpressionParser parser = new SpelExpressionParser(); + + final StandardEvaluationContext context = new StandardEvaluationContext(env.getAttributes()); + context.addPropertyAccessor(new MapAccessor()); + + return parser.parseExpression(f).getValue(context, Boolean.class); + }; + } + +} diff --git a/libs/pom.xml b/libs/pom.xml index e7256d31..20ba7d51 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -19,6 +19,7 @@ dnet-is-common dnet-is-services dnet-data-services + dnet-wf-service diff --git a/pom.xml b/pom.xml index b7839b62..f66fa57e 100644 --- a/pom.xml +++ b/pom.xml @@ -347,8 +347,8 @@ maven-compiler-plugin ${maven.compiler.plugin.version} - 1.8 - 1.8 + 17 + 17 ${project.build.sourceEncoding} @@ -362,7 +362,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.2.1 attach-sources @@ -377,7 +377,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M4 + 3.0.0-M9 true @@ -385,7 +385,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.5.0 true none @@ -394,7 +394,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.0 + 3.5.0 @@ -451,7 +451,7 @@ UTF-8 UTF-8 - 3.6.0 + 3.11.0 1.8 2.14.0 7.1.0