Merge pull request 'new-is-app-with-new-msro' (#13) from new-is-app-with-new-msro into new-is-app

Reviewed-on: #13
This commit is contained in:
Michele Artini 2023-03-21 12:15:25 +01:00
commit 8c3ae35dc4
108 changed files with 4025 additions and 271 deletions

View File

@ -20,6 +20,12 @@
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-wf-service</artifactId>
<version>${project.version}</version>
</dependency>
<dependency> <dependency>
<groupId>eu.dnetlib.dhp</groupId> <groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-data-services</artifactId> <artifactId>dnet-data-services</artifactId>

View File

@ -20,7 +20,11 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
@Configuration @Configuration
@EnableTransactionManagement @EnableTransactionManagement
@EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = { @EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = {
"eu.dnetlib.is", "eu.dnetlib.data.mdstore", "eu.dnetlib.msro" "eu.dnetlib.is",
"eu.dnetlib.common",
"eu.dnetlib.manager.history",
"eu.dnetlib.manager.wf",
"eu.dnetlib.data.mdstore"
}) })
public class MainDBConfig { public class MainDBConfig {
@ -38,7 +42,7 @@ public class MainDBConfig {
@Qualifier("mainDataSource") final DataSource ds) { @Qualifier("mainDataSource") final DataSource ds) {
return builder return builder
.dataSource(ds) .dataSource(ds)
.packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.data.mdstore.model") .packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model")
.persistenceUnit("is") .persistenceUnit("is")
.build(); .build();
} }

View File

@ -21,7 +21,6 @@ public class ZeppelinAjaxController extends AbstractDnetController {
@GetMapping("/templates") @GetMapping("/templates")
public List<String> getTemplates() throws MDStoreManagerException { public List<String> getTemplates() throws MDStoreManagerException {
try { try {
// if (zeppelinClient.get)
return zeppelinClient.listTemplates(); return zeppelinClient.listTemplates();
} catch (final Throwable e) { } catch (final Throwable e) {
throw new MDStoreManagerException("Zeppelin is unreachable", e); throw new MDStoreManagerException("Zeppelin is unreachable", e);

View File

@ -40,9 +40,9 @@ public class DsmAjaxController extends AbstractDnetController {
private ProtocolService protocolService; private ProtocolService protocolService;
@GetMapping("/browsableFields") @GetMapping("/browsableFields")
public List<KeyValue> browsableFields() { public List<KeyValue<String>> browsableFields() {
return Arrays.stream(DsmBrowsableFields.values()) return Arrays.stream(DsmBrowsableFields.values())
.map(f -> new KeyValue(f.name(), f.desc)) .map(f -> new KeyValue<>(f.name(), f.desc))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

View File

@ -0,0 +1,41 @@
package eu.dnetlib.is.email;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.common.model.EmailTemplate;
import eu.dnetlib.notifications.mail.EmailService;
@RestController
@RequestMapping("/ajax/templates/email")
public class EmailTemplateController extends AbstractDnetController {
@Autowired
protected EmailService emailService;
@GetMapping("/")
public List<EmailTemplate> listEmailTemplates() {
return emailService.listEmailTemplates();
}
@PostMapping("/")
public List<EmailTemplate> saveEmailTemplate(@RequestBody final EmailTemplate email) {
emailService.saveEmailTemplate(email);
return emailService.listEmailTemplates();
}
@DeleteMapping("/{id}")
public List<EmailTemplate> deleteEmailTemplate(@PathVariable final String id) {
emailService.deleteEmailTemplate(id);
return emailService.listEmailTemplates();
}
}

View File

@ -1,6 +1,6 @@
package eu.dnetlib.is.importer; package eu.dnetlib.is.importer;
import java.util.Date; import java.time.LocalDateTime;
import javax.transaction.Transactional; import javax.transaction.Transactional;
@ -47,7 +47,7 @@ public class OldProfilesImporter {
final Document doc = DocumentHelper.parseText(xml); final Document doc = DocumentHelper.parseText(xml);
final String id = StringUtils.substringBefore(doc.valueOf("//RESOURCE_IDENTIFIER/@value"), "_"); final String id = StringUtils.substringBefore(doc.valueOf("//RESOURCE_IDENTIFIER/@value"), "_");
final Date now = new Date(); final LocalDateTime now = LocalDateTime.now();
final SimpleResource res = new SimpleResource(); final SimpleResource res = new SimpleResource();
res.setId(id); res.setId(id);

View File

@ -1,11 +1,13 @@
package eu.dnetlib.is.importer; package eu.dnetlib.is.importer;
import java.io.File; import java.io.File;
import java.util.Date; import java.time.Instant;
import java.time.LocalDateTime;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.TimeZone;
import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
@ -17,8 +19,8 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository; import eu.dnetlib.manager.history.model.WfProcessExecution;
import eu.dnetlib.msro.model.history.WfProcessExecution; import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository;
@Service @Service
public class WfHistoryImporter { public class WfHistoryImporter {
@ -53,8 +55,10 @@ public class WfHistoryImporter {
wf.setDsApi(node.get("dataprovider:interface").asText()); wf.setDsApi(node.get("dataprovider:interface").asText());
} }
wf.setStartDate(new Date(NumberUtils.toLong(node.get("system:startDate").asText()))); wf.setStartDate(LocalDateTime
wf.setEndDate(new Date(NumberUtils.toLong(node.get("system:endDate").asText()))); .ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:startDate").asText())), TimeZone.getDefault().toZoneId()));
wf.setEndDate(LocalDateTime.ofInstant(Instant.ofEpochMilli(NumberUtils.toLong(node.get("system:endDate").asText())), TimeZone.getDefault().toZoneId()));
if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) { if (BooleanUtils.toBoolean(node.get("system:isCompletedSuccessfully").asText())) {
wf.setStatus("success"); wf.setStatus("success");

View File

@ -56,30 +56,30 @@ public class InfoAjaxController extends AbstractDnetController {
return res; return res;
} }
private InfoSection<KeyValue> jvm() { private InfoSection<KeyValue<String>> jvm() {
final InfoSection<KeyValue> jvm = new InfoSection<>("JVM"); final InfoSection<KeyValue<String>> jvm = new InfoSection<>("JVM");
jvm.getData().add(new KeyValue("JVM Name", mxbean.getVmName())); jvm.getData().add(new KeyValue<>("JVM Name", mxbean.getVmName()));
jvm.getData().add(new KeyValue("JVM Vendor", mxbean.getVmVendor())); jvm.getData().add(new KeyValue<>("JVM Vendor", mxbean.getVmVendor()));
jvm.getData().add(new KeyValue("JVM Version", mxbean.getVmVersion())); jvm.getData().add(new KeyValue<>("JVM Version", mxbean.getVmVersion()));
jvm.getData().add(new KeyValue("JVM Spec Name", mxbean.getSpecName())); jvm.getData().add(new KeyValue<>("JVM Spec Name", mxbean.getSpecName()));
jvm.getData().add(new KeyValue("JVM Spec Vendor", mxbean.getSpecVendor())); jvm.getData().add(new KeyValue<>("JVM Spec Vendor", mxbean.getSpecVendor()));
jvm.getData().add(new KeyValue("JVM Spec Version", mxbean.getSpecVersion())); jvm.getData().add(new KeyValue<>("JVM Spec Version", mxbean.getSpecVersion()));
jvm.getData().add(new KeyValue("Running JVM Name", mxbean.getName())); jvm.getData().add(new KeyValue<>("Running JVM Name", mxbean.getName()));
jvm.getData().add(new KeyValue("Management Spec Version", mxbean.getManagementSpecVersion())); jvm.getData().add(new KeyValue<>("Management Spec Version", mxbean.getManagementSpecVersion()));
return jvm; return jvm;
} }
private InfoSection<KeyValue> args() { private InfoSection<KeyValue<String>> args() {
final InfoSection<KeyValue> libs = new InfoSection<>("Arguments"); final InfoSection<KeyValue<String>> libs = new InfoSection<>("Arguments");
libs.getData().add(new KeyValue("Input arguments", StringUtils.join(mxbean.getInputArguments(), " "))); libs.getData().add(new KeyValue<>("Input arguments", StringUtils.join(mxbean.getInputArguments(), " ")));
return libs; return libs;
} }
private List<InfoSection<KeyValue>> props() { private List<InfoSection<KeyValue<?>>> props() {
final List<InfoSection<KeyValue>> res = new ArrayList<>(); final List<InfoSection<KeyValue<?>>> res = new ArrayList<>();
configurableEnvironment.getPropertySources().forEach(ps -> { configurableEnvironment.getPropertySources().forEach(ps -> {
final InfoSection<KeyValue> section = new InfoSection<>("Properties: " + ps.getName()); final InfoSection<KeyValue<?>> section = new InfoSection<>("Properties: " + ps.getName());
addAllProperties(section, ps); addAllProperties(section, ps);
res.add(section); res.add(section);
}); });
@ -87,13 +87,13 @@ public class InfoAjaxController extends AbstractDnetController {
return res; return res;
} }
private void addAllProperties(final InfoSection<KeyValue> res, final PropertySource<?> ps) { private void addAllProperties(final InfoSection<KeyValue<?>> res, final PropertySource<?> ps) {
if (ps instanceof CompositePropertySource) { if (ps instanceof CompositePropertySource) {
final CompositePropertySource cps = (CompositePropertySource) ps; final CompositePropertySource cps = (CompositePropertySource) ps;
cps.getPropertySources().forEach(x -> addAllProperties(res, x)); cps.getPropertySources().forEach(x -> addAllProperties(res, x));
} else if (ps instanceof EnumerablePropertySource<?>) { } else if (ps instanceof EnumerablePropertySource<?>) {
final EnumerablePropertySource<?> eps = (EnumerablePropertySource<?>) ps; final EnumerablePropertySource<?> eps = (EnumerablePropertySource<?>) ps;
Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue(k, eps.getProperty(k)))); Arrays.asList(eps.getPropertyNames()).forEach(k -> res.getData().add(new KeyValue<>(k, eps.getProperty(k))));
} else {} } else {}
} }

View File

@ -1,11 +1,11 @@
package eu.dnetlib.is.info; package eu.dnetlib.is.info;
public class KeyValue { public class KeyValue<T> {
private final String k; private final String k;
private final Object v; private final T v;
public KeyValue(final String k, final Object v) { public KeyValue(final String k, final T v) {
this.k = k; this.k = k;
this.v = v; this.v = v;
} }
@ -14,7 +14,7 @@ public class KeyValue {
return k; return k;
} }
public Object getV() { public T getV() {
return v; return v;
} }

View File

@ -25,11 +25,12 @@ public class ResourceAjaxController extends AbstractResourceController {
@PostMapping("/") @PostMapping("/")
public SimpleResource newResource(@RequestParam final String name, public SimpleResource newResource(@RequestParam final String name,
@RequestParam final String type, @RequestParam final String type,
@RequestParam(required = false, defaultValue = "") final String subtype,
@RequestParam(required = false, defaultValue = "") final String description, @RequestParam(required = false, defaultValue = "") final String description,
@RequestParam final String content) @RequestParam final String content)
throws InformationServiceException { throws InformationServiceException {
return service.saveNewResource(name, type, description, content); return service.saveNewResource(name, type, subtype, description, content);
} }
@DeleteMapping("/{resId}") @DeleteMapping("/{resId}")

View File

@ -0,0 +1,36 @@
package eu.dnetlib.manager.wf;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.manager.history.WorkflowLogger;
import eu.dnetlib.manager.history.model.WfProcessExecution;
@RestController
@RequestMapping("/ajax/wf_history")
public class WfHistoryAjaxController extends AbstractDnetController {
@Autowired
private WorkflowLogger logger;
@GetMapping("/")
public List<WfProcessExecution> history(
@RequestParam(required = true) final int total,
@RequestParam(required = false) final Long from,
@RequestParam(required = false) final Long to) {
return logger.history(total, from, to);
}
@GetMapping("/{processId}")
public WfProcessExecution getProcessExecution(@PathVariable final String processId) {
return logger.getProcessExecution(processId);
}
}

View File

@ -0,0 +1,41 @@
package eu.dnetlib.manager.wf;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.is.info.KeyValue;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
@RestController
@RequestMapping("/ajax/wf_instances")
public class WfInstancesController extends AbstractDnetController {
private WorkflowManagerService wfManagerService;
@GetMapping("/instance/{id}")
public WorkflowInstance getWfInstance(@PathVariable final String id) throws Exception {
return wfManagerService.findWorkflowInstance(id);
}
@GetMapping("/search")
public List<KeyValue<String>> listWfInstances(@RequestParam final String section) throws Exception {
return wfManagerService.streamWfInstancesBySection(section)
.map(x -> new KeyValue<>(x.getId(), x.getName()))
.collect(Collectors.toList());
}
@GetMapping("/sections")
public List<KeyValue<Long>> listWfFamilies() throws Exception {
return wfManagerService.streamSections()
.map(x -> new KeyValue<>(x.getValue(), x.getCount()))
.collect(Collectors.toList());
}
}

View File

@ -1,49 +0,0 @@
package eu.dnetlib.msro.history;
import java.util.Date;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.msro.history.repository.WfProcessExecutionRepository;
import eu.dnetlib.msro.model.history.WfProcessExecution;
@RestController
@RequestMapping("/ajax/wfs")
public class WfHistoryAjaxController {
@Autowired
private WfProcessExecutionRepository wfProcessExecutionRepository;
@Deprecated
public static final int MAX_NUMBER_OF_RECENT_WFS = 100;
@GetMapping("/")
public List<WfProcessExecution> history(
@RequestParam(required = true) final int total,
@RequestParam(required = false) final Long from,
@RequestParam(required = false) final Long to) {
if (from == null && to == null) {
return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList();
} else if (from == null) {
return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(0), new Date(to));
} else if (to == null) {
return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date());
} else {
return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(new Date(from), new Date(to));
}
}
@GetMapping("/{processId}")
public WfProcessExecution getProcess(@PathVariable final String processId) {
return wfProcessExecutionRepository.findById(processId).get();
}
}

View File

@ -3,6 +3,8 @@ server.port=8280
server.public_url = server.public_url =
server.public_desc = API Base URL server.public_desc = API Base URL
dnet.configuration.infrastructure = LOCAL DEV
spring.profiles.active=dev spring.profiles.active=dev
maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-pom.xml maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-is-application/effective-pom.xml
@ -57,5 +59,12 @@ dhp.mdstore-manager.hadoop.zeppelin.login =
dhp.mdstore-manager.hadoop.zeppelin.password = dhp.mdstore-manager.hadoop.zeppelin.password =
dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager dhp.mdstore-manager.hadoop.zeppelin.name-prefix = mdstoreManager
# Email Configuration
dnet.configuration.mail.sender.email =
dnet.configuration.mail.sender.name =
dnet.configuration.mail.cc =
dnet.configuration.mail.smtp.host =
dnet.configuration.mail.smtp.port = 25
dnet.configuration.mail.smtp.user =
dnet.configuration.mail.smtp.password =

View File

@ -9,6 +9,8 @@ import { ContextViewerComponent, ContextsComponent } from './contexts/contexts.c
import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/dsm.component'; import { DsmSearchComponent, DsmResultsComponent, DsmApiComponent } from './dsm/dsm.component';
import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component'; import { MdstoreInspectorComponent, MdstoresComponent } from './mdstores/mdstores.component';
import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component';
import { EmailsComponent } from './emails/emails.component';
import { WfInstancesComponent } from './wf-instances/wf-instances.component';
const routes: Routes = [ const routes: Routes = [
{ path: "", redirectTo: 'info', pathMatch: 'full' }, { path: "", redirectTo: 'info', pathMatch: 'full' },
@ -17,6 +19,8 @@ const routes: Routes = [
{ path: "adv_resources/context", component: ContextsComponent }, { path: "adv_resources/context", component: ContextsComponent },
{ path: "adv_resources/vocabulary", component: VocabulariesComponent }, { path: "adv_resources/vocabulary", component: VocabulariesComponent },
{ path: "adv_resources/protocol", component: ProtocolsComponent }, { path: "adv_resources/protocol", component: ProtocolsComponent },
{ path: "adv_resources/email", component: EmailsComponent },
{ path: "wfs", component: WfInstancesComponent },
{ path: "wf_history", component: WfHistoryComponent }, { path: "wf_history", component: WfHistoryComponent },
{ path: "ctx_viewer", component: ContextViewerComponent }, { path: "ctx_viewer", component: ContextViewerComponent },
{ path: "voc_editor", component: VocabularyEditorComponent }, { path: "voc_editor", component: VocabularyEditorComponent },

View File

@ -36,6 +36,9 @@ import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
import { SpinnerHttpInterceptor } from './common/spinner.service'; import { SpinnerHttpInterceptor } from './common/spinner.service';
import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component'; import { MdstoresComponent, MdstoreInspectorComponent, MDStoreVersionsDialog, AddMDStoreDialog } from './mdstores/mdstores.component';
import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component'; import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.component';
import { EmailDialog, EmailsComponent } from './emails/emails.component';
import { WfInstancesComponent } from './wf-instances/wf-instances.component';
import { MatTabsModule } from '@angular/material/tabs';
@NgModule({ @NgModule({
declarations: [ declarations: [
@ -66,7 +69,10 @@ import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.componen
MdstoreInspectorComponent, MdstoreInspectorComponent,
MDStoreVersionsDialog, MDStoreVersionsDialog,
AddMDStoreDialog, AddMDStoreDialog,
CleanerTesterComponent CleanerTesterComponent,
EmailsComponent,
EmailDialog,
WfInstancesComponent
], ],
imports: [ imports: [
BrowserModule, BrowserModule,
@ -95,7 +101,8 @@ import { CleanerTesterComponent } from './cleaner-tester/cleaner-tester.componen
ReactiveFormsModule, ReactiveFormsModule,
MatSnackBarModule, MatSnackBarModule,
MatPaginatorModule, MatPaginatorModule,
MatProgressSpinnerModule MatProgressSpinnerModule,
MatTabsModule
], ],
providers: [{ providers: [{
provide: HTTP_INTERCEPTORS, provide: HTTP_INTERCEPTORS,

View File

@ -54,6 +54,7 @@ export interface SimpleResource {
id: string, id: string,
name: string, name: string,
type: string, type: string,
subtype?: string,
description?: string, description?: string,
creationDate?: string, creationDate?: string,
modificationDate?: string modificationDate?: string
@ -198,3 +199,29 @@ export interface MDStoreRecord {
dateOfTransformation: string, dateOfTransformation: string,
provenance: any provenance: any
} }
export interface EmailTemplate {
id: string,
description: string,
subject: string,
message: string
}
export interface WfInstance {
id: string,
details: Map<string, string>,
priority: number,
dsId?: string,
dsName?: string,
apiId?: string,
enabled: boolean,
configured: boolean,
schedulingEnabled: boolean,
cronExpression?: string,
cronMinInterval?: number,
workflow: string,
destroyWf?: string,
systemParams: Map<string, string>,
userParams: Map<string, string>
}

View File

@ -1,6 +1,6 @@
import { Injectable } from '@angular/core'; import { Injectable } from '@angular/core';
import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http';
import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord } from './is.model'; import { Page, DsmConf, ResourceType, Protocol, WfHistoryEntry, SimpleResource, Context, ContextNode, Vocabulary, VocabularyTerm, KeyValue, BrowseTerm, Datasource, MDStore, MDStoreVersion, MDStoreRecord, EmailTemplate } from './is.model';
import { FormGroup } from '@angular/forms'; import { FormGroup } from '@angular/forms';
import { MatSnackBar } from '@angular/material/snack-bar'; import { MatSnackBar } from '@angular/material/snack-bar';
@ -72,11 +72,12 @@ export class ISService {
}); });
} }
addSimpleResource(name: string, type: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void { addSimpleResource(name: string, type: string, subtype: string, description: string, content: string, onSuccess: Function, relatedForm?: FormGroup): void {
const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded') const headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded')
let body = new HttpParams() let body = new HttpParams()
.set('name', name) .set('name', name)
.set('type', type) .set('type', type)
.set('subtype', subtype)
.set('description', description) .set('description', description)
.set('content', content); .set('content', content);
this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({ this.client.post<void>('/ajax/resources/', body, { headers: headers }).subscribe({
@ -98,7 +99,7 @@ export class ISService {
if (from && from > 0) { params = params.append('from', from); } if (from && from > 0) { params = params.append('from', from); }
if (to && to > 0) { params = params.append('to', to); } if (to && to > 0) { params = params.append('to', to); }
this.client.get<WfHistoryEntry[]>('/ajax/wfs/', { params: params }).subscribe({ this.client.get<WfHistoryEntry[]>('/ajax/wf_history/', { params: params }).subscribe({
next: data => onSuccess(data), next: data => onSuccess(data),
error: error => this.showError(error) error: error => this.showError(error)
}); });
@ -328,6 +329,47 @@ export class ISService {
}); });
} }
loadEmailTemplates(onSuccess: Function): void {
this.client.get<void>('./ajax/templates/email/').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
saveEmailTemplate(email: EmailTemplate, onSuccess: Function, relatedForm?: FormGroup): void {
this.client.post<void>('./ajax/templates/email/', email).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error, relatedForm)
});
}
deleteEmailTemplate(id: string, onSuccess: Function): void {
this.client.delete<void>('./ajax/templates/email/' + encodeURIComponent(id)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadWfIntancesSections(onSuccess: Function): void {
this.client.get<void>('./ajax/wf_instances/sections').subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadWfIntances(section: string, onSuccess: Function): void {
this.client.get<void>('./ajax/wf_instances/search?section=' + encodeURIComponent(section)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
loadWfIntance(id: string, onSuccess: Function): void {
this.client.get<void>('./ajax/wf_instances/instance/' + encodeURIComponent(id)).subscribe({
next: data => onSuccess(data),
error: error => this.showError(error)
});
}
private showError(error: any, form?: FormGroup) { private showError(error: any, form?: FormGroup) {

View File

@ -0,0 +1,41 @@
<form [formGroup]="emailForm" (ngSubmit)="onSubmit()">
<h1 mat-dialog-title *ngIf="emailForm.get('id')?.value">Edit Email Template</h1>
<h1 mat-dialog-title *ngIf="!emailForm.get('id')?.value">New Email Template</h1>
<div mat-dialog-content>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;" *ngIf="emailForm.get('id')?.value">
<mat-label>ID</mat-label>
<input matInput formControlName="id" readonly="readonly" />
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Description</mat-label>
<input matInput formControlName="description" />
<mat-error *ngIf="emailForm.get('description')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Email: subject</mat-label>
<input matInput formControlName="subject" />
<mat-error *ngIf="emailForm.get('subject')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Email: message</mat-label>
<textarea matInput formControlName="message" required rows="16" style="font-size: 0.8em;"></textarea>
<mat-error *ngIf="emailForm.get('message')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field>
</div>
<div mat-dialog-actions>
<button mat-stroked-button color="primary" type="submit" [disabled]="!emailForm.valid">Submit</button>
<button mat-stroked-button color="primary" mat-dialog-close>Close</button>
<mat-error *ngIf="emailForm.errors?.['serverError']">
{{ emailForm.errors?.['serverError'] }}
</mat-error>
</div>
</form>

View File

@ -0,0 +1,42 @@
<h2>Email Templates</h2>
<button mat-stroked-button color="primary" (click)="openAddEmailTemplateDialog()">
<mat-icon fontIcon="add"></mat-icon>
create a new template
</button>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%; margin-top: 10px;">
<mat-label>Filter</mat-label>
<input matInput (keyup)="applyFilter($event)" placeholder="Filter..." #input />
</mat-form-field>
<table mat-table [dataSource]="emailsDatasource" matSort class="mat-elevation-z8">
<ng-container matColumnDef="id">
<th mat-header-cell *matHeaderCellDef style="width: 25%;" mat-sort-header sortActionDescription="Sort by ID"> Id
</th>
<td mat-cell *matCellDef="let element">
<a (click)="openEditEmailTemplateDialog(element)">{{element.id}}</a>
</td>
</ng-container>
<ng-container matColumnDef="description">
<th mat-header-cell *matHeaderCellDef mat-sort-header sortActionDescription="Sort by Description"> Description </th>
<td mat-cell *matCellDef="let element"> {{element.description}} </td>
</ng-container>
<ng-container matColumnDef="buttons">
<th mat-header-cell *matHeaderCellDef style="text-align: right;" style="width: 20%"></th>
<td mat-cell *matCellDef="let element" class="table-buttons">
<button mat-stroked-button color="warn" (click)="deleteEmailTemplate(element)">delete</button>
</td>
</ng-container>
<tr mat-header-row *matHeaderRowDef="colums"></tr>
<tr mat-row *matRowDef="let row; columns: colums;"></tr>
<!-- Row shown when there is no matching data. -->
<tr class="mat-row" *matNoDataRow>
<td class="mat-cell" colspan="4" style="padding: 0 16px;">No data matching the filter "{{input.value}}"</td>
</tr>
</table>

View File

@ -0,0 +1,98 @@
import { AfterViewInit, Component, Inject, OnInit, ViewChild } from '@angular/core';
import { FormControl, FormGroup, Validators } from '@angular/forms';
import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { MatSort } from '@angular/material/sort';
import { MatTableDataSource } from '@angular/material/table';
import { ActivatedRoute } from '@angular/router';
import { EmailTemplate } from '../common/is.model';
import { ISService } from '../common/is.service';
@Component({
selector: 'app-emails',
templateUrl: './emails.component.html',
styleUrls: ['./emails.component.css']
})
export class EmailsComponent implements OnInit, AfterViewInit {
emailsDatasource: MatTableDataSource<EmailTemplate> = new MatTableDataSource<EmailTemplate>([]);
colums: string[] = ['id', 'description', 'buttons'];
@ViewChild(MatSort) sort: MatSort | undefined
searchText: string = '';
constructor(public service: ISService, public route: ActivatedRoute, public dialog: MatDialog) { }
ngOnInit() { this.reload() }
ngAfterViewInit() { if (this.sort) this.emailsDatasource.sort = this.sort; }
reload() { this.service.loadEmailTemplates((data: EmailTemplate[]) => this.emailsDatasource.data = data); }
applyFilter(event: Event) {
const filterValue = (event.target as HTMLInputElement).value.trim().toLowerCase();
this.emailsDatasource.filter = filterValue;
}
openAddEmailTemplateDialog(): void {
const dialogRef = this.dialog.open(EmailDialog, {
data: {
id: '',
description: '',
subject: '',
message: ''
},
width: '80%'
});
dialogRef.afterClosed().subscribe(result => {
if (result) this.reload();
});
}
openEditEmailTemplateDialog(email: EmailTemplate): void {
const dialogRef = this.dialog.open(EmailDialog, {
data: email,
width: '80%'
});
dialogRef.afterClosed().subscribe(result => {
if (result) this.reload();
});
}
deleteEmailTemplate(email: EmailTemplate) {
if (confirm('Are you sure?')) {
this.service.deleteEmailTemplate(email.id, (data: void) => this.reload());
}
}
}
@Component({
selector: 'email-dialog',
templateUrl: './email-dialog.html',
styleUrls: ['./emails.component.css']
})
export class EmailDialog {
emailForm = new FormGroup({
id: new FormControl(''),
description: new FormControl('', [Validators.required]),
subject: new FormControl('', [Validators.required]),
message: new FormControl('', [Validators.required])
});
constructor(public dialogRef: MatDialogRef<EmailDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
this.emailForm.get('id')?.setValue(data.id);
this.emailForm.get('description')?.setValue(data.description);
this.emailForm.get('subject')?.setValue(data.subject);
this.emailForm.get('message')?.setValue(data.message);
}
onSubmit(): void {
const email = Object.assign({}, this.data, this.emailForm.value);
this.service.saveEmailTemplate(email, (data: void) => this.dialogRef.close(1), this.emailForm);
}
onNoClick(): void {
this.dialogRef.close();
}
}

View File

@ -51,6 +51,15 @@
</div> </div>
</mat-expansion-panel> </mat-expansion-panel>
<mat-expansion-panel>
<mat-expansion-panel-header>
<mat-panel-title>Workflows</mat-panel-title>
</mat-expansion-panel-header>
<div>
<a class="menu-item" routerLink="wfs">Workflows</a>
</div>
</mat-expansion-panel>
<mat-expansion-panel> <mat-expansion-panel>
<mat-expansion-panel-header> <mat-expansion-panel-header>
<mat-panel-title>Tools</mat-panel-title> <mat-panel-title>Tools</mat-panel-title>

View File

@ -13,9 +13,14 @@
<input matInput readonly value="{{data.type}}" /> <input matInput readonly value="{{data.type}}" />
</mat-form-field> </mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>SubType (optional)</mat-label>
<input matInput formControlName="subtype" />
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;"> <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Name</mat-label> <mat-label>Name</mat-label>
<input matInput formControlName="name" required /> <input matInput formControlName="name" />
<mat-error *ngIf="metadataForm.get('name')?.invalid">This field is <strong>required</strong></mat-error> <mat-error *ngIf="metadataForm.get('name')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field> </mat-form-field>

View File

@ -5,10 +5,15 @@
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;"> <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Name</mat-label> <mat-label>Name</mat-label>
<input matInput formControlName="name" required /> <input matInput formControlName="name" />
<mat-error *ngIf="newResourceForm.get('name')?.invalid">This field is <strong>required</strong></mat-error> <mat-error *ngIf="newResourceForm.get('name')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field> </mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>SubType (optional)</mat-label>
<input matInput formControlName="subtype" />
</mat-form-field>
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;"> <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Description</mat-label> <mat-label>Description</mat-label>
<textarea matInput formControlName="description" rows="2"></textarea> <textarea matInput formControlName="description" rows="2"></textarea>
@ -16,7 +21,7 @@
<mat-form-field appearance="fill" floatLabel="always" style="width: 100%;"> <mat-form-field appearance="fill" floatLabel="always" style="width: 100%;">
<mat-label>Content ({{data.contentType}})</mat-label> <mat-label>Content ({{data.contentType}})</mat-label>
<textarea matInput formControlName="content" required rows="10" style="font-size: 0.8em;"></textarea> <textarea matInput formControlName="content" rows="10" style="font-size: 0.8em;"></textarea>
<mat-error *ngIf="newResourceForm.get('content')?.invalid">This field is <strong>required</strong></mat-error> <mat-error *ngIf="newResourceForm.get('content')?.invalid">This field is <strong>required</strong></mat-error>
</mat-form-field> </mat-form-field>

View File

@ -12,14 +12,18 @@
<mat-card *ngFor="let r of resources | searchFilter: searchText" style="margin-top: 10px;"> <mat-card *ngFor="let r of resources | searchFilter: searchText" style="margin-top: 10px;">
<mat-card-header> <mat-card-header>
<mat-card-title title="{{r.id}}">{{r.name}} <span class="badge-label badge-info" <mat-card-title title="{{r.id}}">
style="font-size: 0.7em;">{{type.contentType}}</span></mat-card-title> {{r.name}}
<span class="badge-label badge-warning" style="font-size: 0.7em;" *ngIf="r.subtype">{{r.subtype}}</span>
<span class="badge-label badge-info" style="font-size: 0.7em;">{{type.contentType}}</span>
</mat-card-title>
</mat-card-header> </mat-card-header>
<mat-card-content> <mat-card-content>
<p>{{r.description}}</p> <p>{{r.description}}</p>
<p class="muted small"> <p class="muted small">
<b>Id:</b> {{r.id}}<br /> <b>Creation date:</b> {{r.creationDate}}<br /> <b>Modification date:</b> <b>Id:</b> {{r.id}}<br />
{{r.modificationDate}} <b>Creation date:</b> {{r.creationDate}}<br />
<b>Modification date:</b> {{r.modificationDate}}
</p> </p>
</mat-card-content> </mat-card-content>
<mat-card-actions> <mat-card-actions>

View File

@ -3,7 +3,7 @@ import { ISService } from '../common/is.service';
import { ActivatedRoute } from '@angular/router'; import { ActivatedRoute } from '@angular/router';
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { ResourceType, SimpleResource } from '../common/is.model'; import { ResourceType, SimpleResource } from '../common/is.model';
import { FormControl, FormGroup } from '@angular/forms'; import { FormControl, FormGroup, Validators } from '@angular/forms';
@Component({ @Component({
selector: 'app-resources', selector: 'app-resources',
@ -115,12 +115,16 @@ export class ResContentDialog {
}) })
export class ResMetadataDialog { export class ResMetadataDialog {
metadataForm = new FormGroup({ metadataForm = new FormGroup({
name: new FormControl(''), name: new FormControl('', [Validators.required]),
subtype: new FormControl(''),
description: new FormControl('') description: new FormControl('')
}); });
constructor(public dialogRef: MatDialogRef<ResMetadataDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { constructor(public dialogRef: MatDialogRef<ResMetadataDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) {
this.metadataForm.get('name')?.setValue(data.name); this.metadataForm.get('name')?.setValue(data.name);
if (data.subtype) {
this.metadataForm.get('subtype')?.setValue(data.subtype);
}
if (data.description) { if (data.description) {
this.metadataForm.get('description')?.setValue(data.description); this.metadataForm.get('description')?.setValue(data.description);
} }
@ -143,9 +147,10 @@ export class ResMetadataDialog {
}) })
export class ResCreateNewDialog { export class ResCreateNewDialog {
newResourceForm = new FormGroup({ newResourceForm = new FormGroup({
name: new FormControl(''), name: new FormControl('', [Validators.required]),
subtype: new FormControl(''),
description: new FormControl(''), description: new FormControl(''),
content: new FormControl('') content: new FormControl('', [Validators.required])
}); });
constructor(public dialogRef: MatDialogRef<ResCreateNewDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { } constructor(public dialogRef: MatDialogRef<ResCreateNewDialog>, @Inject(MAT_DIALOG_DATA) public data: any, public service: ISService) { }
@ -153,10 +158,11 @@ export class ResCreateNewDialog {
onSubmit(): void { onSubmit(): void {
let name: string = this.newResourceForm.get('name')?.value!; let name: string = this.newResourceForm.get('name')?.value!;
let type: string = this.data.id!; let type: string = this.data.id!;
let subtype: string = this.newResourceForm.get('subtype')?.value!;
let description: string = this.newResourceForm.get('description')?.value!; let description: string = this.newResourceForm.get('description')?.value!;
let content: string = this.newResourceForm.get('content')?.value!; let content: string = this.newResourceForm.get('content')?.value!;
this.service.addSimpleResource(name, type, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm); this.service.addSimpleResource(name, type, subtype, description, content, (data: void) => this.dialogRef.close(1), this.newResourceForm);
} }
onNoClick(): void { onNoClick(): void {
this.dialogRef.close(); this.dialogRef.close();

View File

@ -137,8 +137,6 @@ export class VocabularyEditorComponent implements OnInit, AfterViewInit {
} }
} }
} }
@Component({ @Component({

View File

@ -0,0 +1,7 @@
<h2>Workflow Instances</h2>
<mat-tab-group animationDuration="0ms">
<mat-tab label="First">Content 1</mat-tab>
<mat-tab label="Second">Content 2</mat-tab>
<mat-tab label="Third">Content 3</mat-tab>
</mat-tab-group>

View File

@ -0,0 +1,10 @@
import { Component } from '@angular/core';
@Component({
selector: 'app-wf-instances',
templateUrl: './wf-instances.component.html',
styleUrls: ['./wf-instances.component.css']
})
export class WfInstancesComponent {
}

View File

@ -1,7 +1,8 @@
package eu.dnetlib.data.mdstore; package eu.dnetlib.data.mdstore;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -112,9 +113,9 @@ public class MDStoreService {
private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) { private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) {
final MDStoreVersion v = new MDStoreVersion(); final MDStoreVersion v = new MDStoreVersion();
final Date now = new Date(); final LocalDateTime now = LocalDateTime.now();
final String versionId = md.getId() + "-" + now.getTime(); final String versionId = md.getId() + "-" + now.toEpochSecond(ZoneOffset.UTC);
v.setId(versionId); v.setId(versionId);
v.setMdstore(md.getId()); v.setMdstore(md.getId());
v.setLastUpdate(null); v.setLastUpdate(null);
@ -189,7 +190,7 @@ public class MDStoreService {
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v)); mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
v.setWriting(false); v.setWriting(false);
v.setSize(size); v.setSize(size);
v.setLastUpdate(new Date()); v.setLastUpdate(LocalDateTime.now());
mdstoreVersionRepository.save(v); mdstoreVersionRepository.save(v);
return v; return v;
@ -258,7 +259,7 @@ public class MDStoreService {
md.setLayout(layout); md.setLayout(layout);
md.setType(type); md.setType(type);
md.setInterpretation(interpretation); md.setInterpretation(interpretation);
md.setCreationDate(new Date()); md.setCreationDate(LocalDateTime.now());
md.setDatasourceName(dsName); md.setDatasourceName(dsName);
md.setDatasourceId(dsId); md.setDatasourceId(dsId);
md.setApiId(apiId); md.setApiId(apiId);

View File

@ -1,8 +1,8 @@
package eu.dnetlib.data.mdstore.backends; package eu.dnetlib.data.mdstore.backends;
import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -49,8 +49,8 @@ public class MockBackend implements MDStoreBackend {
rec.setOriginalId("mck-" + i); rec.setOriginalId("mck-" + i);
rec.setId("mock________::mck-" + i); rec.setId("mock________::mck-" + i);
rec.setBody("<RECORD>" + i + "</RECORD>"); rec.setBody("<RECORD>" + i + "</RECORD>");
rec.setDateOfCollection(new Date().getTime()); rec.setDateOfCollection(Instant.now().toEpochMilli());
rec.setDateOfTransformation(new Date().getTime()); rec.setDateOfTransformation(Instant.now().toEpochMilli());
rec.setEncoding("XML"); rec.setEncoding("XML");
rec.setProvenance(MOCK_PROVENANCE); rec.setProvenance(MOCK_PROVENANCE);
list.add(rec); list.add(rec);

View File

@ -0,0 +1,61 @@
package eu.dnetlib.common.model;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
@Entity
@Table(name = "emails")
public class EmailTemplate implements Serializable {
private static final long serialVersionUID = -8424437958140000626L;
@Id
@Column(name = "id")
private String id;
@Column(name = "description")
private String description;
@Column(name = "subject")
private String subject;
@Column(name = "message")
private String message;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getSubject() {
return subject;
}
public void setSubject(final String subject) {
this.subject = subject;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
}

View File

@ -1,7 +1,7 @@
package eu.dnetlib.data.mdstore.model; package eu.dnetlib.data.mdstore.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.LocalDateTime;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
@ -12,8 +12,6 @@ import javax.persistence.EnumType;
import javax.persistence.Enumerated; import javax.persistence.Enumerated;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
@ -55,8 +53,7 @@ public class MDStore implements Serializable {
private Map<String, Object> params = new LinkedHashMap<>(); private Map<String, Object> params = new LinkedHashMap<>();
@Column(name = "creation_date") @Column(name = "creation_date")
@Temporal(TemporalType.TIMESTAMP) private LocalDateTime creationDate;
private Date creationDate;
@Override @Override
public int hashCode() { public int hashCode() {
@ -143,11 +140,11 @@ public class MDStore implements Serializable {
this.params = params; this.params = params;
} }
public Date getCreationDate() { public LocalDateTime getCreationDate() {
return creationDate; return creationDate;
} }
public void setCreationDate(final Date creationDate) { public void setCreationDate(final LocalDateTime creationDate) {
this.creationDate = creationDate; this.creationDate = creationDate;
} }

View File

@ -1,7 +1,7 @@
package eu.dnetlib.data.mdstore.model; package eu.dnetlib.data.mdstore.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.LocalDateTime;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
@ -10,8 +10,6 @@ import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
@ -36,8 +34,7 @@ public class MDStoreVersion implements Serializable {
private int readCount = 0; private int readCount = 0;
@Column(name = "lastupdate") @Column(name = "lastupdate")
@Temporal(TemporalType.TIMESTAMP) private LocalDateTime lastUpdate;
private Date lastUpdate;
@Column(name = "size") @Column(name = "size")
private long size = 0; private long size = 0;
@ -78,11 +75,11 @@ public class MDStoreVersion implements Serializable {
this.readCount = readCount; this.readCount = readCount;
} }
public Date getLastUpdate() { public LocalDateTime getLastUpdate() {
return lastUpdate; return lastUpdate;
} }
public void setLastUpdate(final Date lastUpdate) { public void setLastUpdate(final LocalDateTime lastUpdate) {
this.lastUpdate = lastUpdate; this.lastUpdate = lastUpdate;
} }

View File

@ -1,7 +1,7 @@
package eu.dnetlib.data.mdstore.model; package eu.dnetlib.data.mdstore.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.LocalDateTime;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
@ -12,8 +12,6 @@ import javax.persistence.EnumType;
import javax.persistence.Enumerated; import javax.persistence.Enumerated;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
@ -54,12 +52,10 @@ public class MDStoreWithInfo implements Serializable {
private String currentVersion; private String currentVersion;
@Column(name = "creation_date") @Column(name = "creation_date")
@Temporal(TemporalType.TIMESTAMP) private LocalDateTime creationDate;
private Date creationDate;
@Column(name = "lastupdate") @Column(name = "lastupdate")
@Temporal(TemporalType.TIMESTAMP) private LocalDateTime lastUpdate;
private Date lastUpdate;
@Column(name = "size") @Column(name = "size")
private long size = 0; private long size = 0;
@ -143,19 +139,19 @@ public class MDStoreWithInfo implements Serializable {
this.currentVersion = currentVersion; this.currentVersion = currentVersion;
} }
public Date getCreationDate() { public LocalDateTime getCreationDate() {
return creationDate; return creationDate;
} }
public void setCreationDate(final Date creationDate) { public void setCreationDate(final LocalDateTime creationDate) {
this.creationDate = creationDate; this.creationDate = creationDate;
} }
public Date getLastUpdate() { public LocalDateTime getLastUpdate() {
return lastUpdate; return lastUpdate;
} }
public void setLastUpdate(final Date lastUpdate) { public void setLastUpdate(final LocalDateTime lastUpdate) {
this.lastUpdate = lastUpdate; this.lastUpdate = lastUpdate;
} }

View File

@ -2,6 +2,7 @@ package eu.dnetlib.dsm.model;
import java.io.Serializable; import java.io.Serializable;
import java.sql.Date; import java.sql.Date;
import java.time.LocalDateTime;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
@ -89,7 +90,7 @@ public class Datasource implements Serializable {
private String registeredby; private String registeredby;
private Date registrationdate; private LocalDateTime registrationdate;
private String subjects; private String subjects;
@ -99,10 +100,10 @@ public class Datasource implements Serializable {
private Boolean consentTermsOfUse; private Boolean consentTermsOfUse;
@Column(name = "consenttermsofusedate") @Column(name = "consenttermsofusedate")
private Date consentTermsOfUseDate; private LocalDateTime consentTermsOfUseDate;
@Column(name = "lastconsenttermsofusedate") @Column(name = "lastconsenttermsofusedate")
private Date lastConsentTermsOfUseDate; private LocalDateTime lastConsentTermsOfUseDate;
@Column(name = "fulltextdownload") @Column(name = "fulltextdownload")
private Boolean fullTextDownload; private Boolean fullTextDownload;
@ -401,11 +402,11 @@ public class Datasource implements Serializable {
this.registeredby = registeredby; this.registeredby = registeredby;
} }
public Date getRegistrationdate() { public LocalDateTime getRegistrationdate() {
return registrationdate; return registrationdate;
} }
public void setRegistrationdate(final Date registrationdate) { public void setRegistrationdate(final LocalDateTime registrationdate) {
this.registrationdate = registrationdate; this.registrationdate = registrationdate;
} }
@ -433,19 +434,19 @@ public class Datasource implements Serializable {
this.consentTermsOfUse = consentTermsOfUse; this.consentTermsOfUse = consentTermsOfUse;
} }
public Date getConsentTermsOfUseDate() { public LocalDateTime getConsentTermsOfUseDate() {
return consentTermsOfUseDate; return consentTermsOfUseDate;
} }
public void setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { public void setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate; this.consentTermsOfUseDate = consentTermsOfUseDate;
} }
public Date getLastConsentTermsOfUseDate() { public LocalDateTime getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate; return lastConsentTermsOfUseDate;
} }
public void setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { public void setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
} }

View File

@ -1,7 +1,7 @@
package eu.dnetlib.dsm.model; package eu.dnetlib.dsm.model;
import java.io.Serializable; import java.io.Serializable;
import java.sql.Date; import java.time.LocalDateTime;
import java.util.Set; import java.util.Set;
import javax.persistence.CascadeType; import javax.persistence.CascadeType;
@ -34,7 +34,7 @@ public class Organization implements Serializable {
private String country; private String country;
private String collectedfrom; private String collectedfrom;
private Date dateofcollection; private LocalDateTime dateofcollection;
private String provenanceaction; private String provenanceaction;
@ManyToMany(cascade = { @ManyToMany(cascade = {
@ -98,11 +98,11 @@ public class Organization implements Serializable {
this.collectedfrom = collectedfrom; this.collectedfrom = collectedfrom;
} }
public Date getDateofcollection() { public LocalDateTime getDateofcollection() {
return dateofcollection; return dateofcollection;
} }
public void setDateofcollection(final Date dateofcollection) { public void setDateofcollection(final LocalDateTime dateofcollection) {
this.dateofcollection = dateofcollection; this.dateofcollection = dateofcollection;
} }

View File

@ -0,0 +1,14 @@
package eu.dnetlib.errors;
public class WorkflowManagerException extends Exception {
private static final long serialVersionUID = -9067581185191425823L;
public WorkflowManagerException(final String message, final Throwable cause) {
super(message, cause);
}
public WorkflowManagerException(final String message) {
super(message);
}
}

View File

@ -1,14 +1,12 @@
package eu.dnetlib.is.model.resource; package eu.dnetlib.is.model.resource;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.LocalDateTime;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@Entity @Entity
@Table(name = "resources") @Table(name = "resources")
@ -26,16 +24,17 @@ public class SimpleResource implements Serializable {
@Column(name = "type") @Column(name = "type")
private String type; private String type;
@Column(name = "subtype")
private String subtype;
@Column(name = "description") @Column(name = "description")
private String description; private String description;
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "creation_date") @Column(name = "creation_date")
private Date creationDate; private LocalDateTime creationDate;
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "modification_date") @Column(name = "modification_date")
private Date modificationDate; private LocalDateTime modificationDate;
public String getId() { public String getId() {
return id; return id;
@ -61,6 +60,14 @@ public class SimpleResource implements Serializable {
this.type = type; this.type = type;
} }
public String getSubtype() {
return subtype;
}
public void setSubtype(final String subtype) {
this.subtype = subtype;
}
public String getDescription() { public String getDescription() {
return description; return description;
} }
@ -69,19 +76,19 @@ public class SimpleResource implements Serializable {
this.description = description; this.description = description;
} }
public Date getCreationDate() { public LocalDateTime getCreationDate() {
return creationDate; return creationDate;
} }
public void setCreationDate(final Date creationDate) { public void setCreationDate(final LocalDateTime creationDate) {
this.creationDate = creationDate; this.creationDate = creationDate;
} }
public Date getModificationDate() { public LocalDateTime getModificationDate() {
return modificationDate; return modificationDate;
} }
public void setModificationDate(final Date modificationDate) { public void setModificationDate(final LocalDateTime modificationDate) {
this.modificationDate = modificationDate; this.modificationDate = modificationDate;
} }

View File

@ -1,15 +1,13 @@
package eu.dnetlib.msro.model.history; package eu.dnetlib.manager.history.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.time.LocalDateTime;
import java.util.Map; import java.util.Map;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
import javax.persistence.Id; import javax.persistence.Id;
import javax.persistence.Table; import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.hibernate.annotations.Type; import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef; import org.hibernate.annotations.TypeDef;
@ -32,6 +30,9 @@ public class WfProcessExecution implements Serializable {
@Column(name = "process_id") @Column(name = "process_id")
private String processId; private String processId;
@Column(name = "wf_instance_id")
private String wfInstanceId;
@Column(name = "name") @Column(name = "name")
private String name; private String name;
@ -41,13 +42,11 @@ public class WfProcessExecution implements Serializable {
@Column(name = "status") @Column(name = "status")
private String status; private String status;
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "start_date") @Column(name = "start_date")
private Date startDate; private LocalDateTime startDate;
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "end_date") @Column(name = "end_date")
private Date endDate; private LocalDateTime endDate;
@Column(name = "ds_id") @Column(name = "ds_id")
private String dsId; private String dsId;
@ -70,6 +69,14 @@ public class WfProcessExecution implements Serializable {
this.processId = processId; this.processId = processId;
} }
public String getWfInstanceId() {
return wfInstanceId;
}
public void setWfInstanceId(final String wfInstanceId) {
this.wfInstanceId = wfInstanceId;
}
public String getName() { public String getName() {
return name; return name;
} }
@ -94,19 +101,19 @@ public class WfProcessExecution implements Serializable {
this.status = status; this.status = status;
} }
public Date getStartDate() { public LocalDateTime getStartDate() {
return startDate; return startDate;
} }
public void setStartDate(final Date startDate) { public void setStartDate(final LocalDateTime startDate) {
this.startDate = startDate; this.startDate = startDate;
} }
public Date getEndDate() { public LocalDateTime getEndDate() {
return endDate; return endDate;
} }
public void setEndDate(final Date endDate) { public void setEndDate(final LocalDateTime endDate) {
this.endDate = endDate; this.endDate = endDate;
} }

View File

@ -0,0 +1,5 @@
package eu.dnetlib.manager.wf.model;
public enum NotificationCondition {
ALWAYS, NEVER, ONLY_SUCCESS, ONLY_FAILED
}

View File

@ -0,0 +1,269 @@
package eu.dnetlib.manager.wf.model;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.env.Environment;
public class WorkflowGraph implements Serializable {
private static final long serialVersionUID = 5919290887480115842L;
public List<WfParam> parameters;
public List<Node> graph;
public List<Node> getGraph() {
return graph;
}
public void setGraph(final List<Node> graph) {
this.graph = graph;
}
public List<WfParam> getParameters() {
return parameters;
}
public void setParameters(final List<WfParam> parameters) {
this.parameters = parameters;
}
class WfParam implements Serializable {
private static final long serialVersionUID = 5885589803738655166L;
private String name;
private String description;
private String type;
private String defaultValue;
private boolean required;
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getDefaultValue() {
return defaultValue;
}
public void setDefaultValue(final String defaultValue) {
this.defaultValue = defaultValue;
}
public boolean isRequired() {
return required;
}
public void setRequired(final boolean required) {
this.required = required;
}
}
public class Node implements Serializable {
private static final long serialVersionUID = -3695762832959801906L;
private static final String regExRef = "\\$\\{(\\w*)\\}";
private String name;
private String type;
private boolean isStart = false;
private boolean isJoin = false;
private List<Arc> arcs;
private List<NodeParam> input;
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public boolean isStart() {
return isStart;
}
public void setStart(final boolean isStart) {
this.isStart = isStart;
}
public boolean isJoin() {
return isJoin;
}
public void setJoin(final boolean isJoin) {
this.isJoin = isJoin;
}
public List<Arc> getArcs() {
return arcs;
}
public void setArcs(final List<Arc> arcs) {
this.arcs = arcs;
}
public List<NodeParam> getInput() {
return input;
}
public void setInput(final List<NodeParam> input) {
this.input = input;
}
public Map<String, String> findEnvParams() {
return input.stream()
.filter(p -> StringUtils.isNotBlank(p.getEnv()))
.collect(Collectors.toMap(NodeParam::getName, NodeParam::getEnv));
}
public Map<String, Object> calculateInitialParams(final Map<String, String> globalParams, final Environment environment) {
final Map<String, Object> map = new HashMap<>();
input.stream()
.filter(p -> StringUtils.isBlank(p.getEnv()))
.forEach(p -> map.put(p.getName(), calculateSimpleValue(p, globalParams, environment)));
return map;
}
private Object calculateSimpleValue(final NodeParam p, final Map<String, String> globalParams, final Environment environment) {
String value = p.getValue();
final String ref = p.getRef();
final String prop = p.getProperty();
if (StringUtils.isNotBlank(ref) && StringUtils.isNotBlank(globalParams.get(ref))) {
return globalParams.get(ref);
} else if (StringUtils.isNotBlank(value)) {
final Matcher matcher = Pattern.compile(regExRef, Pattern.MULTILINE).matcher(value);
while (matcher.find()) {
final String rName = matcher.group(1);
final String rValue = globalParams.get(rName);
if (StringUtils.isBlank(rValue)) { return null; }
value = value.replaceAll(Pattern.quote(matcher.group(0)), rValue);
System.out.println("NEW VALUE " + value);
}
return value;
} else if (StringUtils.isNotBlank(prop)) {
return environment.getProperty(prop);
} else {
return null;
}
}
}
public class Arc implements Serializable {
private static final long serialVersionUID = 7866138976929522262L;
private String to;
private String condition;
public String getTo() {
return to;
}
public void setTo(final String to) {
this.to = to;
}
public String getCondition() {
return condition;
}
public void setCondition(final String condition) {
this.condition = condition;
}
}
class NodeParam implements Serializable {
private static final long serialVersionUID = 7815785723401725707L;
private String name;
private String value;
private String ref;
private String property;
private String env;
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(final String value) {
this.value = value;
}
public String getRef() {
return ref;
}
public void setRef(final String ref) {
this.ref = ref;
}
public String getProperty() {
return property;
}
public void setProperty(final String property) {
this.property = property;
}
public String getEnv() {
return env;
}
public void setEnv(final String env) {
this.env = env;
}
}
}

View File

@ -0,0 +1,232 @@
package eu.dnetlib.manager.wf.model;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.Map;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Transient;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.hibernate.annotations.TypeDefs;
import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
import com.vladmihalcea.hibernate.type.json.JsonStringType;
@Entity
@Table(name = "workflow_instances")
@TypeDefs({
@TypeDef(name = "json", typeClass = JsonStringType.class),
@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
})
public class WorkflowInstance implements Serializable {
private static final long serialVersionUID = 7503841966138333044L;
@Id
@Column(name = "id")
private String id;
@Column(name = "name")
private String name;
@Column(name = "section")
private String section;
@Type(type = "jsonb")
@Column(name = "details", columnDefinition = "jsonb")
private Map<String, String> details = new LinkedHashMap<>();
@Column(name = "priority")
private Integer priority;
@Column(name = "dsid")
private String dsId;
@Column(name = "dsname")
private String dsName;
@Column(name = "apiid")
private String apiId;
@Column(name = "enabled")
private boolean enabled;
@Column(name = "configured")
private boolean configured;
@Column(name = "scheduling_enabled")
private boolean schedulingEnabled;
@Column(name = "scheduling_cron")
private String cronExpression;
@Column(name = "scheduling_min_interval")
private int cronMinInterval;
@Column(name = "workflow")
private String workflow;
@Column(name = "destroy_wf")
private String destroyWf;
@Type(type = "jsonb")
@Column(name = "system_params", columnDefinition = "jsonb")
private Map<String, String> systemParams = new LinkedHashMap<>();
@Type(type = "jsonb")
@Column(name = "user_params", columnDefinition = "jsonb")
private Map<String, String> userParams = new LinkedHashMap<>();
@Transient
private String parentId;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getSection() {
return section;
}
public void setSection(final String section) {
this.section = section;
}
public Map<String, String> getDetails() {
return details;
}
public void setDetails(final Map<String, String> details) {
this.details = details;
}
public Integer getPriority() {
return priority;
}
public void setPriority(final Integer priority) {
this.priority = priority;
}
public String getDsId() {
return dsId;
}
public void setDsId(final String dsId) {
this.dsId = dsId;
}
public String getDsName() {
return dsName;
}
public void setDsName(final String dsName) {
this.dsName = dsName;
}
public String getApiId() {
return apiId;
}
public void setApiId(final String apiId) {
this.apiId = apiId;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(final boolean enabled) {
this.enabled = enabled;
}
public boolean isConfigured() {
return configured;
}
public void setConfigured(final boolean configured) {
this.configured = configured;
}
public boolean isSchedulingEnabled() {
return schedulingEnabled;
}
public void setSchedulingEnabled(final boolean schedulingEnabled) {
this.schedulingEnabled = schedulingEnabled;
}
public String getCronExpression() {
return cronExpression;
}
public void setCronExpression(final String cronExpression) {
this.cronExpression = cronExpression;
}
public int getCronMinInterval() {
return cronMinInterval;
}
public void setCronMinInterval(final int cronMinInterval) {
this.cronMinInterval = cronMinInterval;
}
public String getWorkflow() {
return workflow;
}
public void setWorkflow(final String workflow) {
this.workflow = workflow;
}
public String getDestroyWf() {
return destroyWf;
}
public void setDestroyWf(final String destroyWf) {
this.destroyWf = destroyWf;
}
public Map<String, String> getSystemParams() {
return systemParams;
}
public void setSystemParams(final Map<String, String> systemParams) {
this.systemParams = systemParams;
}
public Map<String, String> getUserParams() {
return userParams;
}
public void setUserParams(final Map<String, String> userParams) {
this.userParams = userParams;
}
public String getParentId() {
return parentId;
}
public void setParentId(final String parentId) {
this.parentId = parentId;
}
}

View File

@ -0,0 +1,5 @@
package eu.dnetlib.manager.wf.model;
public class WorkflowParamDesc {
}

View File

@ -0,0 +1,68 @@
package eu.dnetlib.manager.wf.model;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.Table;
@Entity
@Table(name = "workflow_subscriptions")
@IdClass(WorkflowSubscriptionPK.class)
public class WorkflowSubscription implements Serializable {
private static final long serialVersionUID = -3662770213782581404L;
@Id
@Column(name = "wf_instance_id")
private String wfInstanceId;
@Id
@Column(name = "condition")
@Enumerated(EnumType.STRING)
private NotificationCondition condition;
@Id
@Column(name = "email")
private String email;
@Column(name = "message_id")
private String messageId;
public String getWfInstanceId() {
return wfInstanceId;
}
public void setWfInstanceId(final String wfInstanceId) {
this.wfInstanceId = wfInstanceId;
}
public NotificationCondition getCondition() {
return condition;
}
public void setCondition(final NotificationCondition condition) {
this.condition = condition;
}
public String getEmail() {
return email;
}
public void setEmail(final String email) {
this.email = email;
}
public String getMessageId() {
return messageId;
}
public void setMessageId(final String messageId) {
this.messageId = messageId;
}
}

View File

@ -0,0 +1,39 @@
package eu.dnetlib.manager.wf.model;
import java.io.Serializable;
public class WorkflowSubscriptionPK implements Serializable {
private static final long serialVersionUID = -7569690774071644848L;
private String wfInstanceId;
private NotificationCondition condition;
private String email;
public String getWfInstanceId() {
return wfInstanceId;
}
public void setWfInstanceId(final String wfInstanceId) {
this.wfInstanceId = wfInstanceId;
}
public NotificationCondition getCondition() {
return condition;
}
public void setCondition(final NotificationCondition condition) {
this.condition = condition;
}
public String getEmail() {
return email;
}
public void setEmail(final String email) {
this.email = email;
}
}

View File

@ -0,0 +1,13 @@
package eu.dnetlib.utils;
public interface CountedValue {
public String getValue();
public void setValue();
public long getCount();
public void setCount(final long count);
}

View File

@ -1,9 +1,11 @@
package eu.dnetlib.utils; package eu.dnetlib.utils;
import java.io.StringWriter; import java.io.StringWriter;
import java.text.SimpleDateFormat; import java.time.Instant;
import java.util.Date; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale; import java.util.Locale;
import java.util.TimeZone;
public class DateUtils { public class DateUtils {
@ -12,7 +14,7 @@ public class DateUtils {
private static final long HOUR = MINUTE * 60; private static final long HOUR = MINUTE * 60;
private static final long DAY = HOUR * 24; private static final long DAY = HOUR * 24;
private static final long YEAR = DAY * 365; private static final long YEAR = DAY * 365;
private static final SimpleDateFormat ISO8601FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US); private static final DateTimeFormatter ISO8601FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ", Locale.getDefault());
public static String elapsedTime(long t) { public static String elapsedTime(long t) {
final StringWriter a = new StringWriter(); final StringWriter a = new StringWriter();
@ -78,7 +80,13 @@ public class DateUtils {
} }
public static String calculate_ISO8601(final long l) { public static String calculate_ISO8601(final long l) {
String result = ISO8601FORMAT.format(new Date(l));
final LocalDateTime time =
LocalDateTime.ofInstant(Instant.ofEpochMilli(l), TimeZone
.getDefault()
.toZoneId());
String result = time.format(ISO8601FORMAT);
// convert YYYYMMDDTHH:mm:ss+HH00 into YYYYMMDDTHH:mm:ss+HH:00 // convert YYYYMMDDTHH:mm:ss+HH00 into YYYYMMDDTHH:mm:ss+HH:00
// - note the added colon for the Timezone // - note the added colon for the Timezone
result = result.substring(0, result.length() - 2) + ":" + result.substring(result.length() - 2); result = result.substring(0, result.length() - 2) + ":" + result.substring(result.length() - 2);
@ -89,4 +97,8 @@ public class DateUtils {
return Long.toString(Math.floorDiv(d, n)); return Long.toString(Math.floorDiv(d, n));
} }
public static long now() {
return Instant.now().toEpochMilli();
}
} }

View File

@ -0,0 +1,10 @@
package eu.dnetlib.utils;
public interface Stoppable {
void stop();
void resume();
public StoppableDetails getStopDetails();
}

View File

@ -0,0 +1,33 @@
package eu.dnetlib.utils;
public class StoppableDetails {
public enum StopStatus {
STOPPED,
STOPPING,
RUNNING
};
private final String title;
private final String description;
private final StopStatus status;
public StoppableDetails(final String title, final String description, final StopStatus status) {
this.title = title;
this.description = description;
this.status = status;
}
public String getTitle() {
return title;
}
public String getDescription() {
return description;
}
public StopStatus getStatus() {
return status;
}
}

View File

@ -106,6 +106,7 @@ CREATE INDEX ON context_cat_concepts_lvl_2 (parent);
CREATE TABLE wf_history ( CREATE TABLE wf_history (
process_id text PRIMARY KEY, process_id text PRIMARY KEY,
wf_instance_id text NOT NULL,
name text NOT NULL, name text NOT NULL,
family text NOT NULL, family text NOT NULL,
status text NOT NULL, status text NOT NULL,
@ -130,7 +131,8 @@ INSERT INTO resource_types(id, name, content_type) VALUES
('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'), ('transformation_rule_legacy', 'Transformation Rules (legacy)', 'text/plain'),
('cleaning_rule', 'Cleaning Rules', 'application/xml'), ('cleaning_rule', 'Cleaning Rules', 'application/xml'),
('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml') ('hadoop_job_configuration', 'Hadoop Job Configurations', 'application/xml')
('dedup_configuration', 'Dedup Configurations', 'application/json'); ('dedup_configuration', 'Dedup Configurations', 'application/json')
('workflow', 'Workflows', 'application/json');
CREATE TABLE resources ( CREATE TABLE resources (
id text PRIMARY KEY, id text PRIMARY KEY,
@ -138,6 +140,7 @@ CREATE TABLE resources (
description text, description text,
content text NOT NULL DEFAULT '', content text NOT NULL DEFAULT '',
type text NOT NULL REFERENCES resource_types(id), type text NOT NULL REFERENCES resource_types(id),
subtype text,
creation_date timestamp NOT NULL DEFAULT now(), creation_date timestamp NOT NULL DEFAULT now(),
modification_date timestamp NOT NULL DEFAULT now() modification_date timestamp NOT NULL DEFAULT now()
); );
@ -177,6 +180,14 @@ CREATE VIEW resource_types_view AS (
count(*) AS count, count(*) AS count,
false AS simple false AS simple
FROM protocols FROM protocols
) UNION ALL (
SELECT
'email' AS id,
'Email templates' AS name,
'text/plain' AS content_type,
count(*) AS count,
false AS simple
FROM emails
); );
CREATE TABLE mdstores ( CREATE TABLE mdstores (
@ -241,3 +252,40 @@ GROUP BY md.id,
v1.lastupdate, v1.lastupdate,
v1.size; v1.size;
-- Email Templates
CREATE TABLE emails (
id text PRIMARY KEY,
description text NOT NULL,
subject text NOT NULL,
message text NOT NULL
);
-- Workflows
CREATE TABLE workflow_instances (
id text PRIMARY KEY,
name text NOT NULL,
section text,
details jsonb NOT NULL DEFAULT '{}',
priority int,
dsid text,
dsname text,
apiid text,
enabled boolean NOT NULL DEFAULT false,
configured boolean NOT NULL DEFAULT false,
scheduling_enabled boolean NOT NULL DEFAULT false,
scheduling_cron text,
scheduling_min_interval int,
workflow text REFERENCES resource(id),
destroy_wf text REFERENCES resource(id),
system_params jsonb NOT NULL DEFAULT '{}',
user_params jsonb NOT NULL DEFAULT '{}'
);
CREATE TABLE workflow_subscriptions (
wf_instance_id text NOT NULL REFERENCES workflow_instances(id),
condition text NOT NULL,
email text NOT NULL,
message_id text NOT NULL REFERENCES emails(id),
PRIMARY KEY (wf_instance_id, condition, email)
);

View File

@ -25,6 +25,11 @@
<artifactId>jaxen</artifactId> <artifactId>jaxen</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<!-- Tests --> <!-- Tests -->
<dependency> <dependency>
<groupId>org.junit.jupiter</groupId> <groupId>org.junit.jupiter</groupId>

View File

@ -0,0 +1,9 @@
package eu.dnetlib.common.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.common.model.EmailTemplate;
public interface EmailTemplateRepository extends JpaRepository<EmailTemplate, String> {
}

View File

@ -9,7 +9,7 @@ import static eu.dnetlib.dsm.utils.DsmMappingUtils.createId;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.sql.Date; import java.time.LocalDate;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
@ -368,7 +368,7 @@ public class DsmService {
private void ensureRegistrationDate(final String dsId) { private void ensureRegistrationDate(final String dsId) {
if (!dsRepository.hasRegistrationdate(dsId)) { if (!dsRepository.hasRegistrationdate(dsId)) {
log.info("setting registration date for datasource: " + dsId); log.info("setting registration date for datasource: " + dsId);
dsRepository.setRegistrationDate(dsId, new Date(System.currentTimeMillis())); dsRepository.setRegistrationDate(dsId, LocalDate.now());
} }
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.sql.Date; import java.time.LocalDateTime;
import java.util.Set; import java.util.Set;
import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect;
@ -27,15 +27,15 @@ public class ApiDetails extends ApiIgnoredProperties {
private Integer lastCollectionTotal; private Integer lastCollectionTotal;
private Date lastCollectionDate; private LocalDateTime lastCollectionDate;
private Integer lastAggregationTotal; private Integer lastAggregationTotal;
private Date lastAggregationDate; private LocalDateTime lastAggregationDate;
private Integer lastDownloadTotal; private Integer lastDownloadTotal;
private Date lastDownloadDate; private LocalDateTime lastDownloadDate;
private String baseurl; private String baseurl;
@ -71,7 +71,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return lastCollectionTotal; return lastCollectionTotal;
} }
public Date getLastCollectionDate() { public LocalDateTime getLastCollectionDate() {
return lastCollectionDate; return lastCollectionDate;
} }
@ -79,7 +79,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return lastAggregationTotal; return lastAggregationTotal;
} }
public Date getLastAggregationDate() { public LocalDateTime getLastAggregationDate() {
return lastAggregationDate; return lastAggregationDate;
} }
@ -87,7 +87,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return lastDownloadTotal; return lastDownloadTotal;
} }
public Date getLastDownloadDate() { public LocalDateTime getLastDownloadDate() {
return lastDownloadDate; return lastDownloadDate;
} }
@ -125,7 +125,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return this; return this;
} }
public ApiDetails setLastCollectionDate(final Date lastCollectionDate) { public ApiDetails setLastCollectionDate(final LocalDateTime lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate; this.lastCollectionDate = lastCollectionDate;
return this; return this;
} }
@ -135,7 +135,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return this; return this;
} }
public ApiDetails setLastAggregationDate(final Date lastAggregationDate) { public ApiDetails setLastAggregationDate(final LocalDateTime lastAggregationDate) {
this.lastAggregationDate = lastAggregationDate; this.lastAggregationDate = lastAggregationDate;
return this; return this;
} }
@ -145,7 +145,7 @@ public class ApiDetails extends ApiIgnoredProperties {
return this; return this;
} }
public ApiDetails setLastDownloadDate(final Date lastDownloadDate) { public ApiDetails setLastDownloadDate(final LocalDateTime lastDownloadDate) {
this.lastDownloadDate = lastDownloadDate; this.lastDownloadDate = lastDownloadDate;
return this; return this;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.sql.Date; import java.time.LocalDate;
import java.util.Set; import java.util.Set;
import javax.persistence.Transient; import javax.persistence.Transient;
@ -48,12 +48,12 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
private String languages; private String languages;
private Date dateofvalidation; private LocalDate dateofvalidation;
@NotBlank @NotBlank
private String eoscDatasourceType; private String eoscDatasourceType;
private Date dateofcollection; private LocalDate dateofcollection;
private String platform; private String platform;
@ -82,9 +82,9 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
private Boolean fullTextDownload; private Boolean fullTextDownload;
private Date consentTermsOfUseDate; private LocalDate consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate; private LocalDate lastConsentTermsOfUseDate;
private Set<OrganizationDetails> organizations; private Set<OrganizationDetails> organizations;
@ -95,7 +95,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
@Deprecated @Deprecated
private String typology; private String typology;
private Date registrationdate; private LocalDate registrationdate;
public String getId() { public String getId() {
return id; return id;
@ -145,7 +145,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return namespaceprefix; return namespaceprefix;
} }
public Date getDateofvalidation() { public LocalDate getDateofvalidation() {
return dateofvalidation; return dateofvalidation;
} }
@ -153,7 +153,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return eoscDatasourceType; return eoscDatasourceType;
} }
public Date getDateofcollection() { public LocalDate getDateofcollection() {
return dateofcollection; return dateofcollection;
} }
@ -277,7 +277,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return this; return this;
} }
public DatasourceDetails setDateofvalidation(final Date dateofvalidation) { public DatasourceDetails setDateofvalidation(final LocalDate dateofvalidation) {
this.dateofvalidation = dateofvalidation; this.dateofvalidation = dateofvalidation;
return this; return this;
} }
@ -287,7 +287,7 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return this; return this;
} }
public DatasourceDetails setDateofcollection(final Date dateofcollection) { public DatasourceDetails setDateofcollection(final LocalDate dateofcollection) {
this.dateofcollection = dateofcollection; this.dateofcollection = dateofcollection;
return this; return this;
} }
@ -367,11 +367,11 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return this; return this;
} }
public Date getConsentTermsOfUseDate() { public LocalDate getConsentTermsOfUseDate() {
return consentTermsOfUseDate; return consentTermsOfUseDate;
} }
public DatasourceDetails setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { public DatasourceDetails setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate; this.consentTermsOfUseDate = consentTermsOfUseDate;
return this; return this;
} }
@ -396,20 +396,20 @@ public class DatasourceDetails extends DatasourceIgnoredProperties {
return this; return this;
} }
public Date getLastConsentTermsOfUseDate() { public LocalDate getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate; return lastConsentTermsOfUseDate;
} }
public DatasourceDetails setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { public DatasourceDetails setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this; return this;
} }
public Date getRegistrationdate() { public LocalDate getRegistrationdate() {
return registrationdate; return registrationdate;
} }
public DatasourceDetails setRegistrationdate(final Date registrationdate) { public DatasourceDetails setRegistrationdate(final LocalDate registrationdate) {
this.registrationdate = registrationdate; this.registrationdate = registrationdate;
return this; return this;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.sql.Date; import java.time.LocalDate;
import java.util.Set; import java.util.Set;
import javax.validation.constraints.Email; import javax.validation.constraints.Email;
@ -57,9 +57,9 @@ public class DatasourceDetailsUpdate {
private Boolean consentTermsOfUse; private Boolean consentTermsOfUse;
private Date consentTermsOfUseDate; private LocalDate consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate; private LocalDate lastConsentTermsOfUseDate;
private Boolean fullTextDownload; private Boolean fullTextDownload;
@ -207,11 +207,11 @@ public class DatasourceDetailsUpdate {
return this; return this;
} }
public Date getConsentTermsOfUseDate() { public LocalDate getConsentTermsOfUseDate() {
return consentTermsOfUseDate; return consentTermsOfUseDate;
} }
public DatasourceDetailsUpdate setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { public DatasourceDetailsUpdate setConsentTermsOfUseDate(final LocalDate consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate; this.consentTermsOfUseDate = consentTermsOfUseDate;
return this; return this;
} }
@ -225,11 +225,11 @@ public class DatasourceDetailsUpdate {
return this; return this;
} }
public Date getLastConsentTermsOfUseDate() { public LocalDate getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate; return lastConsentTermsOfUseDate;
} }
public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { public DatasourceDetailsUpdate setLastConsentTermsOfUseDate(final LocalDate lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this; return this;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.sql.Date; import java.time.LocalDate;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
@ -13,10 +13,10 @@ public abstract class DatasourceIgnoredProperties {
protected String provenanceaction; protected String provenanceaction;
@JsonIgnore @JsonIgnore
protected Date releasestartdate; protected LocalDate releasestartdate;
@JsonIgnore @JsonIgnore
protected Date releaseenddate; protected LocalDate releaseenddate;
@JsonIgnore @JsonIgnore
protected String missionstatementurl; protected String missionstatementurl;
@ -76,19 +76,19 @@ public abstract class DatasourceIgnoredProperties {
this.provenanceaction = provenanceaction; this.provenanceaction = provenanceaction;
} }
public Date getReleasestartdate() { public LocalDate getReleasestartdate() {
return releasestartdate; return releasestartdate;
} }
public void setReleasestartdate(final Date releasestartdate) { public void setReleasestartdate(final LocalDate releasestartdate) {
this.releasestartdate = releasestartdate; this.releasestartdate = releasestartdate;
} }
public Date getReleaseenddate() { public LocalDate getReleaseenddate() {
return releaseenddate; return releaseenddate;
} }
public void setReleaseenddate(final Date releaseenddate) { public void setReleaseenddate(final LocalDate releaseenddate) {
this.releaseenddate = releaseenddate; this.releaseenddate = releaseenddate;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.util.Date; import java.time.LocalDateTime;
import java.util.Set; import java.util.Set;
import javax.validation.constraints.Email; import javax.validation.constraints.Email;
@ -28,7 +28,7 @@ public class DatasourceSnippetExtended {
@Email @Email
private String registeredby; private String registeredby;
private Date registrationdate; private LocalDateTime registrationdate;
private String eoscDatasourceType; private String eoscDatasourceType;
@ -38,9 +38,9 @@ public class DatasourceSnippetExtended {
private Boolean consentTermsOfUse; private Boolean consentTermsOfUse;
private Date consentTermsOfUseDate; private LocalDateTime consentTermsOfUseDate;
private Date lastConsentTermsOfUseDate; private LocalDateTime lastConsentTermsOfUseDate;
private Boolean fullTextDownload; private Boolean fullTextDownload;
@ -89,11 +89,11 @@ public class DatasourceSnippetExtended {
this.registeredby = registeredby; this.registeredby = registeredby;
} }
public Date getRegistrationdate() { public LocalDateTime getRegistrationdate() {
return registrationdate; return registrationdate;
} }
public void setRegistrationdate(final Date registrationdate) { public void setRegistrationdate(final LocalDateTime registrationdate) {
this.registrationdate = registrationdate; this.registrationdate = registrationdate;
} }
@ -118,11 +118,11 @@ public class DatasourceSnippetExtended {
return this; return this;
} }
public Date getConsentTermsOfUseDate() { public LocalDateTime getConsentTermsOfUseDate() {
return consentTermsOfUseDate; return consentTermsOfUseDate;
} }
public DatasourceSnippetExtended setConsentTermsOfUseDate(final Date consentTermsOfUseDate) { public DatasourceSnippetExtended setConsentTermsOfUseDate(final LocalDateTime consentTermsOfUseDate) {
this.consentTermsOfUseDate = consentTermsOfUseDate; this.consentTermsOfUseDate = consentTermsOfUseDate;
return this; return this;
} }
@ -170,11 +170,11 @@ public class DatasourceSnippetExtended {
return this; return this;
} }
public Date getLastConsentTermsOfUseDate() { public LocalDateTime getLastConsentTermsOfUseDate() {
return lastConsentTermsOfUseDate; return lastConsentTermsOfUseDate;
} }
public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final Date lastConsentTermsOfUseDate) { public DatasourceSnippetExtended setLastConsentTermsOfUseDate(final LocalDateTime lastConsentTermsOfUseDate) {
this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate; this.lastConsentTermsOfUseDate = lastConsentTermsOfUseDate;
return this; return this;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.domain; package eu.dnetlib.dsm.domain;
import java.util.Date; import java.time.LocalDateTime;
import java.util.Set; import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
@ -14,7 +14,7 @@ public class OrganizationIgnoredProperties {
protected String collectedfrom; protected String collectedfrom;
@JsonIgnore @JsonIgnore
protected Date dateofcollection; protected LocalDateTime dateofcollection;
@JsonIgnore @JsonIgnore
protected String provenanceaction; protected String provenanceaction;
@ -38,11 +38,11 @@ public class OrganizationIgnoredProperties {
this.collectedfrom = collectedfrom; this.collectedfrom = collectedfrom;
} }
public Date getDateofcollection() { public LocalDateTime getDateofcollection() {
return dateofcollection; return dateofcollection;
} }
public void setDateofcollection(final Date dateofcollection) { public void setDateofcollection(final LocalDateTime dateofcollection) {
this.dateofcollection = dateofcollection; this.dateofcollection = dateofcollection;
} }

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.repository; package eu.dnetlib.dsm.repository;
import java.sql.Date; import java.time.LocalDate;
import java.util.Optional; import java.util.Optional;
import javax.transaction.Transactional; import javax.transaction.Transactional;
@ -64,7 +64,7 @@ public interface DatasourceRepository extends JpaRepository<Datasource, String>,
@Modifying @Modifying
@Transactional @Transactional
@Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1") @Query("update #{#entityName} d set d.registrationdate = ?2 where d.id = ?1")
void setRegistrationDate(String id, Date registrationdate); void setRegistrationDate(String id, LocalDate registrationdate);
@Modifying @Modifying
@Transactional @Transactional

View File

@ -1,6 +1,6 @@
package eu.dnetlib.dsm.utils; package eu.dnetlib.dsm.utils;
import java.sql.Date; import java.time.LocalDateTime;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -98,7 +98,7 @@ public class DsmMappingUtils {
final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId(); final String prefix = StringUtils.isNotBlank(dbe.getNamespaceprefix()) ? dbe.getNamespaceprefix() : dbe.getId();
o.setId(prefix + ID_SEPARATOR + o.getLegalname()); o.setId(prefix + ID_SEPARATOR + o.getLegalname());
if (o.getDateofcollection() == null) { if (o.getDateofcollection() == null) {
o.setDateofcollection(new Date(System.currentTimeMillis())); o.setDateofcollection(LocalDateTime.now());
} }
o.setCollectedfrom(dbe.getCollectedfrom()); o.setCollectedfrom(dbe.getCollectedfrom());
}); });

View File

@ -1,6 +1,6 @@
package eu.dnetlib.is.resource; package eu.dnetlib.is.resource;
import java.util.Date; import java.time.LocalDateTime;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -60,17 +60,19 @@ public class SimpleResourceService {
@Transactional @Transactional
public SimpleResource saveNewResource(final String name, public SimpleResource saveNewResource(final String name,
final String type, final String type,
final String subtype,
final String description, final String description,
final String content) throws InformationServiceException { final String content) throws InformationServiceException {
resourceValidator.validate(type, content); resourceValidator.validate(type, content);
final Date now = new Date(); final LocalDateTime now = LocalDateTime.now();
final SimpleResource res = new SimpleResource(); final SimpleResource res = new SimpleResource();
res.setId(UUID.randomUUID().toString()); res.setId(UUID.randomUUID().toString());
res.setName(name); res.setName(name);
res.setType(type); res.setType(type);
res.setSubtype(subtype);
res.setDescription(description); res.setDescription(description);
res.setCreationDate(now); res.setCreationDate(now);
res.setModificationDate(now); res.setModificationDate(now);

View File

@ -0,0 +1,51 @@
package eu.dnetlib.manager.history;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Service;
import eu.dnetlib.manager.history.model.WfProcessExecution;
import eu.dnetlib.manager.history.repository.WfProcessExecutionRepository;
@Service
public class WorkflowLogger {
@Autowired
private WfProcessExecutionRepository wfProcessExecutionRepository;
public List<WfProcessExecution> history(final int total, final Long from, final Long to) {
if (from == null && to == null) {
return wfProcessExecutionRepository.findAll(PageRequest.of(0, total, Sort.by("endDate").descending())).toList();
} else {
final LocalDateTime fromTime = from != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(from), TimeZone
.getDefault()
.toZoneId()) : LocalDateTime.MIN;
final LocalDateTime toTime = to != null ? LocalDateTime.ofInstant(Instant.ofEpochMilli(to), TimeZone
.getDefault()
.toZoneId()) : LocalDateTime.MAX;
return wfProcessExecutionRepository.findByEndDateBetweenOrderByEndDateDesc(fromTime, toTime);
}
}
public WfProcessExecution getProcessExecution(final String processId) {
return wfProcessExecutionRepository.findById(processId).get();
}
public void saveProcessExecution(final WfProcessExecution pe) {
wfProcessExecutionRepository.save(pe);
}
public Optional<WfProcessExecution> getLastExecutionForInstance(final String id) {
return wfProcessExecutionRepository.findOneByWfInstanceIdOrderByEndDateAsc(id);
}
}

View File

@ -0,0 +1,16 @@
package eu.dnetlib.manager.history.repository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.manager.history.model.WfProcessExecution;
public interface WfProcessExecutionRepository extends JpaRepository<WfProcessExecution, String> {
List<WfProcessExecution> findByEndDateBetweenOrderByEndDateDesc(LocalDateTime start, LocalDateTime end);
Optional<WfProcessExecution> findOneByWfInstanceIdOrderByEndDateAsc(String id);
}

View File

@ -0,0 +1,21 @@
package eu.dnetlib.manager.wf.repository;
import java.util.stream.Stream;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.utils.CountedValue;
public interface WorkflowInstanceRepository extends JpaRepository<WorkflowInstance, String> {
@Query(value = "select section as value, count(*) as count "
+ "from workflow_instances "
+ "group by section "
+ "order by count desc;", nativeQuery = true)
Stream<CountedValue> streamSections();
Stream<WorkflowInstance> findBySection(String section);
}

View File

@ -0,0 +1,13 @@
package eu.dnetlib.manager.wf.repository;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.manager.wf.model.WorkflowSubscription;
import eu.dnetlib.manager.wf.model.WorkflowSubscriptionPK;
public interface WorkflowSubscriptionRepository extends JpaRepository<WorkflowSubscription, WorkflowSubscriptionPK> {
List<WorkflowSubscription> findByWfInstanceId(String wfInstanceId);
}

View File

@ -1,13 +0,0 @@
package eu.dnetlib.msro.history.repository;
import java.util.Date;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import eu.dnetlib.msro.model.history.WfProcessExecution;
public interface WfProcessExecutionRepository extends JpaRepository<WfProcessExecution, String> {
List<WfProcessExecution> findByEndDateBetweenOrderByEndDateDesc(Date start, Date end);
}

View File

@ -0,0 +1,170 @@
package eu.dnetlib.notifications.mail;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import javax.annotation.PostConstruct;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.google.common.base.Splitter;
import eu.dnetlib.common.model.EmailTemplate;
import eu.dnetlib.common.repository.EmailTemplateRepository;
@Service
public class EmailService {
private static final Log log = LogFactory.getLog(EmailService.class);
private final BlockingQueue<Message> queue = new LinkedBlockingQueue<>();
@Value("${dnet.configuration.mail.sender.email}")
private String from;
@Value("${dnet.configuration.mail.sender.name}")
private String fromName;
@Value("${dnet.configuration.mail.cc}")
private String cc;
@Value("${dnet.configuration.mail.smtp.host}")
private String smtpHost;
@Value("${dnet.configuration.mail.smtp.port}")
private final int smtpPort = 587;
@Value("${dnet.configuration.mail.smtp.user}")
private String smtpUser;
@Value("${dnet.configuration.mail.smtp.password}")
private String smtpPassword;
@Value("${server.public_url}")
private String baseUrl;
@Value("${dnet.configuration.infrastructure}")
private String infrastructure;
@Autowired
private EmailTemplateRepository emailTemplateRepository;
@PostConstruct
private void init() {
new Thread(() -> {
while (true) {
try {
final Message message = this.queue.take();
if (message != null) {
try {
log.info("Sending mail...");
Transport.send(message);
log.info("...sent");
} catch (final MessagingException e) {
log.error("Error sending email", e);
this.queue.add(message);
}
}
} catch (final InterruptedException e1) {
throw new RuntimeException(e1);
}
}
}).start();
}
public List<EmailTemplate> listEmailTemplates() {
return emailTemplateRepository.findAll();
}
public void saveEmailTemplate(final EmailTemplate email) {
if (StringUtils.isBlank(email.getId()) || email.getId().length() < 10) {
email.setId("email-" + UUID.randomUUID());
log.info("Saving new email with id: " + email.getId());
}
emailTemplateRepository.save(email);
}
public void deleteEmailTemplate(final String id) {
emailTemplateRepository.deleteById(id);
}
public void sendMail(final String to, final String subject, final String message) {
try {
final Session session = Session.getInstance(obtainProperties(), obtainAuthenticator());
final MimeMessage mimeMessage = new MimeMessage(session);
mimeMessage.setFrom(new InternetAddress(this.from, this.fromName));
mimeMessage.setSubject(subject);
mimeMessage.setContent(message, "text/html; charset=utf-8");
mimeMessage.setSentDate(new Date());
mimeMessage.addRecipient(Message.RecipientType.TO, new InternetAddress(to));
if (this.cc != null && !this.cc.isEmpty()) {
for (final String aCC : Splitter.on(",").omitEmptyStrings().trimResults().split(cc)) {
mimeMessage.addRecipient(Message.RecipientType.CC, new InternetAddress(aCC));
}
}
this.queue.add(mimeMessage);
log.info("Mail to " + to + " in queue");
} catch (final Exception e) {
log.error("Error sending mail", e);
}
}
public void sendStoredMail(final String to, final String emailId, final Map<String, Object> params) {
// TODO use a real template library
emailTemplateRepository.findById(emailId).ifPresent(tmpl -> {
String msg = tmpl.getMessage();
String subject = tmpl.getSubject();
for (final Entry<String, Object> e : params.entrySet()) {
msg = msg.replaceAll("{" + e.getKey() + "}", e.getValue().toString());
subject = subject.replaceAll("{" + e.getKey() + "}", e.getValue().toString());
}
sendMail(to, subject, msg);
});
}
private Properties obtainProperties() {
final Properties props = new Properties();
props.put("mail.transport.protocol", "smtp");
props.put("mail.smtp.host", this.smtpHost);
props.put("mail.smtp.port", this.smtpPort);
props.put("mail.smtp.auth", Boolean.toString(this.smtpUser != null && !this.smtpUser.isEmpty()));
return props;
}
private Authenticator obtainAuthenticator() {
if (this.smtpUser == null || this.smtpUser.isEmpty()) { return null; }
final String user = this.smtpUser;
final String passwd = this.smtpPassword;
return new Authenticator() {
private final PasswordAuthentication authentication = new PasswordAuthentication(user, passwd);
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return this.authentication;
}
};
}
}

View File

@ -0,0 +1,48 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dnet-wf-service</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-is-services</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-data-services</artifactId>
<version>${project.version}</version>
</dependency>
<!-- Tests -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,26 @@
package eu.dnetlib.manager.wf;
import java.util.LinkedHashMap;
import java.util.Map;
public class NodeInfo {
private String name;
private Map<String, String> params = new LinkedHashMap<>();
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public Map<String, String> getParams() {
return params;
}
public void setParams(final Map<String, String> params) {
this.params = params;
}
}

View File

@ -0,0 +1,20 @@
package eu.dnetlib.manager.wf;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Service;
@Service
public class WfManager {
@Autowired
private ApplicationContext applicationContext;
/*
* public Object obtainBaseNode(final String name) { return applicationContext.getBean(name); }
*
* public NodeInfo obtainInfo(final BaseNode<?, ?> node) { final NodeInfo info = new NodeInfo(); final WfNode annotation =
* node.getClass().getAnnotation(WfNode.class); info.setName(annotation.name()); for (final WfParam p : annotation.inputParams()) {
* info.getParams().put(p.name(), p.type()); } return info; }
*/
}

View File

@ -0,0 +1,198 @@
package eu.dnetlib.manager.wf;
import java.util.HashMap;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import javax.annotation.PostConstruct;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dsm.DsmService;
import eu.dnetlib.errors.DsmException;
import eu.dnetlib.errors.WorkflowManagerException;
import eu.dnetlib.is.model.resource.SimpleResource;
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.manager.wf.model.WorkflowGraph;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository;
import eu.dnetlib.manager.wf.workflows.procs.ProcessEngine;
import eu.dnetlib.manager.wf.workflows.procs.ProcessFactory;
import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
import eu.dnetlib.utils.CountedValue;
import eu.dnetlib.utils.Stoppable;
import eu.dnetlib.utils.StoppableDetails;
@Service
public class WorkflowManagerService implements Stoppable {
private static final Log log = LogFactory.getLog(WorkflowManagerService.class);
@Autowired
private ProcessRegistry processRegistry;
@Autowired
private ProcessFactory processFactory;
@Autowired
private ProcessEngine processEngine;
@Autowired
private DsmService dsmService;
@Autowired
private SimpleResourceRepository simpleResourceRepository;
@Autowired
private WorkflowInstanceRepository workflowInstanceRepository;
private boolean paused = false;
@PostConstruct
public void init() {
Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
if (isPaused() || processRegistry.countRunningWfs() >= WorkflowsConstants.MAX_RUNNING_PROCS_SIZE) { return; }
final WorkflowProcess process = processRegistry.nextProcessToStart();
if (process != null) {
processEngine.startProcess(process);
} else {
log.debug("WorkflowProcess queue is empty");
}
}, 10, 10, TimeUnit.SECONDS);
}
public WorkflowInstance findWorkflowInstance(final String id) throws WorkflowManagerException {
return workflowInstanceRepository.findById(id).orElseThrow(() -> new WorkflowManagerException("WF instance not found: " + id));
}
public String startRepoHiWorkflow(final String wfId,
final String dsId,
final String apiId,
final ExecutionCallback<WorkflowProcess> callback)
throws WorkflowManagerException {
if (isPaused()) {
log.warn("Wf " + wfId + " not launched, because WorkflowExecutor is preparing for shutdown");
throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown");
}
try {
final String dsName = dsmService.getDs(dsId).getOfficialname();
final WorkflowInstance instance = new WorkflowInstance();
instance.setId("REPO_HI_" + UUID.randomUUID());
instance.setDetails(new HashMap<>());
instance.setPriority(100);
instance.setDsId(dsId);
instance.setDsName(dsName);
instance.setApiId(apiId);
instance.setEnabled(true);
instance.setConfigured(true);
instance.setSchedulingEnabled(false);
instance.setCronExpression("");
instance.setCronMinInterval(0);
instance.setWorkflow(wfId);
instance.setDestroyWf(null);
instance.setSystemParams(new HashMap<>());
instance.setUserParams(new HashMap<>());
return startWorkflowInstance(instance, callback);
} catch (final DsmException e) {
throw new WorkflowManagerException("Invalid datasource: " + dsId, e);
}
}
public String startWorkflowInstance(final String wfInstanceId,
final String parent,
final ExecutionCallback<WorkflowProcess> callback) throws Exception {
if (isPaused()) {
log.warn("Wf instance " + wfInstanceId + " not launched, because WorkflowExecutor is preparing for shutdown");
throw new WorkflowManagerException("WorkflowExecutor is preparing for shutdown");
}
final WorkflowInstance instance = findWorkflowInstance(wfInstanceId);
return startWorkflowInstance(instance, callback);
}
public String startWorkflowInstance(final WorkflowInstance wfInstance,
final ExecutionCallback<WorkflowProcess> callback)
throws WorkflowManagerException {
if (!wfInstance.isEnabled() || !wfInstance.isConfigured()) {
log.warn("Wf instance " + wfInstance.getId() + " is not ready to start");
throw new WorkflowManagerException("Wf instance " + wfInstance.getId() + " is not ready to start");
}
final SimpleResource wfMetadata = simpleResourceRepository
.findById(wfInstance.getWorkflow())
.filter(r -> r.getType().equals("workflows"))
.orElseThrow(() -> new WorkflowManagerException("WF not found: " + wfInstance.getWorkflow()));
final WorkflowGraph wfGraph = simpleResourceRepository.findContentById(wfMetadata.getId())
.map(s -> {
try {
return new ObjectMapper().readValue(s, WorkflowGraph.class);
} catch (final Exception e) {
return (WorkflowGraph) null;
}
})
.filter(Objects::nonNull)
.orElseThrow(() -> new WorkflowManagerException("Invalid wf: " + wfMetadata.getId()));
final WorkflowProcess process =
processFactory.newProcess(wfMetadata, wfGraph, wfInstance, callback);
return processRegistry.registerProcess(process, wfInstance);
}
@Override
public void stop() {
paused = true;
}
@Override
public void resume() {
paused = false;
}
@Override
public StoppableDetails getStopDetails() {
final int count = processRegistry.countRunningWfs();
final StoppableDetails.StopStatus status =
isPaused() ? count == 0 ? StoppableDetails.StopStatus.STOPPED : StoppableDetails.StopStatus.STOPPING : StoppableDetails.StopStatus.RUNNING;
return new StoppableDetails("D-NET workflow manager", "Running workflows: " + count, status);
}
public ProcessRegistry getProcessRegistry() {
return processRegistry;
}
public boolean isPaused() {
return paused;
}
public void setPaused(final boolean paused) {
this.paused = paused;
}
public Stream<CountedValue> streamSections() {
return workflowInstanceRepository.streamSections();
}
public Stream<WorkflowInstance> streamWfInstancesBySection(final String section) {
return workflowInstanceRepository.findBySection(section);
}
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.manager.wf.annotations;
public enum StreamMimeType {
XML,
JSON,
TEXT,
UNDEFINED
}

View File

@ -0,0 +1,20 @@
package eu.dnetlib.manager.wf.annotations;
public @interface WfNode {
String name();
WfNodeOperation operation();
Class<?> inputStreamType() default void.class;
StreamMimeType inputStreamMimeType() default StreamMimeType.UNDEFINED;
Class<?> outputStreamType() default void.class;
StreamMimeType outputStreamMimeType() default StreamMimeType.UNDEFINED;
WfParam[] inputParams() default {};
WfParam[] outputParams() default {};
}

View File

@ -0,0 +1,11 @@
package eu.dnetlib.manager.wf.annotations;
public enum WfNodeOperation {
CREATE,
DROP,
READ,
WRITE,
PRODUCER,
TRANSFORM,
SETENV
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.manager.wf.annotations;
public @interface WfParam {
String name();
String type();
}

View File

@ -0,0 +1,138 @@
package eu.dnetlib.manager.wf.cron;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.support.CronExpression;
import org.springframework.stereotype.Service;
import eu.dnetlib.manager.history.WorkflowLogger;
import eu.dnetlib.manager.wf.WorkflowManagerService;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.manager.wf.repository.WorkflowInstanceRepository;
import eu.dnetlib.manager.wf.workflows.procs.ProcessRegistry;
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
@Service
public class ScheduledWorkflowLauncher {
private static final Log log = LogFactory.getLog(ScheduledWorkflowLauncher.class);
@Autowired
private WorkflowManagerService wfManagerService;
@Autowired
private ProcessRegistry processRegistry;
@Autowired
private WorkflowInstanceRepository workflowInstanceRepository;
@Autowired
private WorkflowLogger logger;
@Value("${dnet.workflow.scheduler.windowSize:1800000}")
private int windowSize; // 1800000 are 30 minutes
@Scheduled(fixedRateString = "${dnet.workflow.scheduler.fixedRate:900000}") // 900000 are 5 minutes
public void verifySheduledWorkflows() {
log.debug("Verifying scheduled workflows - START");
workflowInstanceRepository.findAll()
.stream()
.filter(WorkflowInstance::isEnabled)
.filter(WorkflowInstance::isConfigured)
.filter(WorkflowInstance::isSchedulingEnabled)
.filter(this::isNotRunning)
.filter(this::isReady)
.forEach(instance -> {
try {
wfManagerService.startWorkflowInstance(instance, null);
} catch (final Exception e) {
log.error("Error launching scheduled wf instance: " + instance.getId(), e);
}
});
log.debug("Verifying scheduled workflows - END");
}
private boolean isReady(final WorkflowInstance instance) {
final LocalDateTime lastExecutionDate = calculateLastExecutionDate(instance.getId());
final LocalDateTime now = LocalDateTime.now();
final String cron = instance.getCronExpression();
if (CronExpression.isValidExpression(cron)) {
final int minInterval = instance.getCronMinInterval(); // in minutes
final boolean res;
if (lastExecutionDate != null) {
final long elapsed = ChronoUnit.MINUTES.between(lastExecutionDate, now);
res = elapsed > minInterval && verifyCron(cron, now);
} else {
res = verifyCron(cron, now);
}
if (log.isDebugEnabled()) {
log.debug("**************************************************************");
log.debug("WORKFLOW INSTANCE ID : " + instance.getId());
log.debug("NOW : " + now);
log.debug("LAST EXECUTION DATE : " + lastExecutionDate);
log.debug("MIN INTERVAL (minutes) : " + minInterval);
log.debug("WINDOW SIZE (ms) : " + windowSize);
log.debug("MUST BE EXECUTED : " + res);
log.debug("**************************************************************");
}
return res;
}
return false;
}
private LocalDateTime calculateLastExecutionDate(final String id) {
return logger.getLastExecutionForInstance(id)
.map(e -> e.getEndDate())
.orElse(LocalDateTime.MIN);
}
private boolean verifyCron(final String cronExpression, final LocalDateTime now) {
try {
final CronExpression cron = CronExpression.parse(cronExpression);
final LocalDateTime date = now.minus(windowSize, ChronoUnit.MINUTES);
final LocalDateTime nextDate = cron.next(date);
if (log.isDebugEnabled()) {
log.debug("NEXT EXECUTION DATE: " + nextDate);
log.debug("FIRED : " + nextDate.isBefore(now));
}
return nextDate.isBefore(now);
} catch (final Exception e) {
log.error("Error calculating next cron event: " + cronExpression, e);
return false;
}
}
private boolean isNotRunning(final WorkflowInstance instance) {
final WorkflowProcess p = processRegistry.findProcsByInstanceId(instance.getId());
if (p != null) {
switch (p.getStatus()) {
case CREATED:
return false;
case EXECUTING:
return false;
default:
break;
}
}
return true;
}
}

View File

@ -0,0 +1,71 @@
package eu.dnetlib.manager.wf.nodes;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.dnetlib.manager.wf.workflows.procs.Token;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
public abstract class AbstractJobNode extends ProcessNode {
private static final Log log = LogFactory.getLog(AbstractJobNode.class);
private final boolean async;
public AbstractJobNode(final boolean async) {
this.async = async;
}
private final ExecutorService executor = Executors.newCachedThreadPool();
@Override
public final void execute(final Token token, final ExecutionCallback<Token> callback) {
try {
log.debug("START NODE: " + getBeanName());
token.setProgressMessage(getNodeName());
beforeStart(token);
if (isAsync()) {
executor.execute(() -> doExecute(token, callback));
} else {
doExecute(token, callback);
}
log.debug("END NODE (SUCCESS): " + getBeanName());
} catch (final Throwable e) {
log.error("got exception while executing workflow node", e);
log.debug("END NODE (FAILED): " + getBeanName());
beforeFailed(token);
callback.onFail(token);
}
}
private final void doExecute(final Token token, final ExecutionCallback<Token> callback) {
execute(token);
beforeCompleted(token);
callback.onSuccess(token);
}
protected abstract void execute(Token token);
public final boolean isAsync() {
return async;
}
protected void beforeStart(final Token token) {
// For optional overwrites
}
protected void beforeCompleted(final Token token) {
// For optional overwrites
}
protected void beforeFailed(final Token token) {
// For optional overwrites
}
}

View File

@ -0,0 +1,29 @@
package eu.dnetlib.manager.wf.nodes;
import java.util.function.Supplier;
import java.util.stream.Stream;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import eu.dnetlib.manager.wf.annotations.StreamMimeType;
import eu.dnetlib.manager.wf.annotations.WfNode;
import eu.dnetlib.manager.wf.annotations.WfNodeOperation;
import eu.dnetlib.manager.wf.annotations.WfParam;
@Component("oai_collect")
@Scope("prototype")
@WfNode(name = "oai_collect", operation = WfNodeOperation.PRODUCER, inputParams = {
@WfParam(name = "configuration", type = "ApiDescriptor.class"),
}, outputStreamType = String.class, outputStreamMimeType = StreamMimeType.XML)
public class CollectOAINode implements Supplier<Stream<String>> {
private String datasourceID;
@Override
public Stream<String> get() {
// TODO Auto-generated method stub
return null;
}
}

View File

@ -0,0 +1,18 @@
package eu.dnetlib.manager.wf.nodes;
import eu.dnetlib.manager.wf.workflows.procs.Token;
/**
* Created by michele on 26/11/15.
*/
public final class DefaultJobNode extends AbstractJobNode {
public DefaultJobNode(final String name) {
super(false);
setNodeName(name);
}
@Override
public void execute(final Token token) {}
}

View File

@ -0,0 +1,96 @@
package eu.dnetlib.manager.wf.nodes;
import java.util.HashMap;
import java.util.UUID;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import eu.dnetlib.manager.wf.WorkflowManagerService;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.manager.wf.workflows.procs.ProcessAware;
import eu.dnetlib.manager.wf.workflows.procs.Token;
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
/**
* Created by michele on 18/11/15.
*/
public class LaunchWorkflowJobNode extends ProcessNode implements ProcessAware {
private static final Log log = LogFactory.getLog(LaunchWorkflowJobNode.class);
private String wfId;
@Autowired
private WorkflowManagerService wfManagerService;
private WorkflowProcess process;
@Override
public final void execute(final Token token, final ExecutionCallback<Token> callback) {
try {
final WorkflowInstance instance = new WorkflowInstance();
instance.setId("CHILD_" + UUID.randomUUID());
instance.setParentId(process.getWfInstanceId());
instance.setDetails(new HashMap<>());
instance.setPriority(100);
instance.setDsId(process.getDsId());
instance.setDsName(process.getDsName());
instance.setApiId(process.getDsInterface());
instance.setEnabled(true);
instance.setConfigured(true);
instance.setSchedulingEnabled(false);
instance.setCronExpression("");
instance.setCronMinInterval(0);
instance.setWorkflow(wfId);
instance.setDestroyWf(null);
instance.setSystemParams(process.getGlobalParams());
instance.setUserParams(new HashMap<>());
final String procId = wfManagerService.startWorkflowInstance(instance, new ExecutionCallback<WorkflowProcess>() {
@Override
public void onSuccess(final WorkflowProcess t) {
log.debug("Child workflow has been completed successfully");
token.release();
callback.onSuccess(token);
}
@Override
public void onFail(final WorkflowProcess t) {
log.error("Child workflow is failed");
token.releaseAsFailed("Child workflow is failed");
callback.onFail(token);
}
});
if (log.isDebugEnabled()) {
log.debug("The child workflow [instance: " + instance.getId() + "] is starting with procId: " + procId);
}
token.setProgressMessage("Launched sub workflow, proc: " + procId);
} catch (final Throwable e) {
log.error("got exception while launching child workflow", e);
callback.onFail(token);
}
}
@Override
public void setProcess(final WorkflowProcess process) {
this.process = process;
}
public String getWfId() {
return wfId;
}
public void setWfId(final String wfId) {
this.wfId = wfId;
}
}

View File

@ -0,0 +1,8 @@
package eu.dnetlib.manager.wf.nodes;
public enum NodeStatus {
CONFIGURED,
NOT_CONFIGURED,
DISABLED,
SYSTEM
}

View File

@ -0,0 +1,38 @@
package eu.dnetlib.manager.wf.nodes;
import org.springframework.beans.factory.BeanNameAware;
import eu.dnetlib.manager.wf.workflows.procs.Token;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
public abstract class ProcessNode implements BeanNameAware {
private String beanName;
private String nodeName;
public abstract void execute(final Token token, ExecutionCallback<Token> callback);
public String getBeanName() {
return this.beanName;
}
@Override
public void setBeanName(final String beanName) {
this.beanName = beanName;
}
public String getNodeName() {
return this.nodeName;
}
public void setNodeName(final String nodeName) {
this.nodeName = nodeName;
}
@Override
public String toString() {
return String.format("[node beanName=%s, name=%s]", this.beanName, this.nodeName);
}
}

View File

@ -0,0 +1,9 @@
package eu.dnetlib.manager.wf.nodes;
public abstract class SimpleJobNode extends AbstractJobNode {
public SimpleJobNode() {
super(false);
}
}

View File

@ -0,0 +1,18 @@
package eu.dnetlib.manager.wf.nodes;
import eu.dnetlib.manager.wf.workflows.procs.Token;
/**
* Created by michele on 26/11/15.
*/
public class SuccessNode extends AbstractJobNode {
public SuccessNode() {
super(false);
setNodeName("success");
}
@Override
protected void execute(final Token token) {}
}

View File

@ -0,0 +1,46 @@
package eu.dnetlib.manager.wf.notification;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import eu.dnetlib.manager.wf.model.NotificationCondition;
import eu.dnetlib.manager.wf.repository.WorkflowSubscriptionRepository;
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess;
import eu.dnetlib.manager.wf.workflows.procs.WorkflowProcess.Status;
import eu.dnetlib.notifications.mail.EmailService;
@Service
public class EmailSender {
private static final Log log = LogFactory.getLog(EmailSender.class);
@Autowired
private WorkflowSubscriptionRepository wfSubscriptionRepository;
@Autowired
private EmailService emailService;
public void sendMails(final WorkflowProcess proc) {
wfSubscriptionRepository.findByWfInstanceId(proc.getWfInstanceId()).forEach(s -> {
if (s.getCondition() == NotificationCondition.ALWAYS ||
s.getCondition() == NotificationCondition.ONLY_FAILED && proc.getStatus() == Status.FAILURE ||
s.getCondition() == NotificationCondition.ONLY_SUCCESS && proc.getStatus() == Status.SUCCESS) {
try {
final Map<String, Object> params = new HashMap<>();
emailService.sendStoredMail(s.getEmail(), s.getMessageId(), params);
} catch (final Exception e) {
log.error("Error sending mail to " + s.getEmail(), e);
}
}
});
}
}

View File

@ -0,0 +1,40 @@
package eu.dnetlib.manager.wf.workflows.graph;
import com.google.common.base.Function;
import eu.dnetlib.manager.wf.workflows.procs.Env;
public class Arc {
private final String from;
private final String to;
private final Function<Env, Boolean> condFunction;
public Arc(final String from, final String to, final Function<Env, Boolean> condFunction) {
this.from = from;
this.to = to;
this.condFunction = condFunction;
}
public String getFrom() {
return this.from;
}
public String getTo() {
return this.to;
}
public boolean isValid(final Env env) {
if (condFunction != null) {
return condFunction.apply(env);
} else {
return true;
}
}
@Override
public String toString() {
return String.format("[ %s -> %s, %s ]", this.from, this.to, this.condFunction != null ? "with cond" : "without cond");
}
}

View File

@ -0,0 +1,84 @@
package eu.dnetlib.manager.wf.workflows.graph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.manager.wf.workflows.procs.Env;
public class Graph {
private final Map<String, GraphNode> nodes = new HashMap<>();
private List<Arc> arcs = new ArrayList<>();
public void addArc(final Arc arc) {
this.arcs.add(arc);
}
public void addNode(final GraphNode node) {
this.nodes.put(node.getName(), node);
}
public Set<String> nodeNames() {
return this.nodes.keySet();
}
public Collection<GraphNode> nodes() {
return this.nodes.values();
}
public GraphNode getNode(final String name) {
return this.nodes.get(name);
}
public List<Arc> getArcs() {
return this.arcs;
}
public void setArcs(final List<Arc> arcs) {
this.arcs = arcs;
}
public Set<GraphNode> startNodes() {
final Set<GraphNode> res = new HashSet<>();
for (final GraphNode n : this.nodes.values()) {
if (n.isStart()) {
res.add(n);
}
}
return res;
}
public Set<GraphNode> nextNodes(final GraphNode current, final Env env) {
return arcs.stream()
.filter(arc -> StringUtils.equals(arc.getFrom(), current.getName()))
.filter(arc -> arc.isValid(env))
.map(arc -> arc.getTo())
.distinct()
.map(to -> nodes.get(to))
.collect(Collectors.toSet());
}
public int getNumberOfIncomingArcs(final GraphNode node) {
int count = 0;
for (final Arc arc : this.arcs) {
if (arc.getTo().equals(node.getName())) {
count++;
}
}
return count;
}
@Override
public String toString() {
return "\n************************\nNodes: " + this.nodes + "\nArcs: " + this.arcs + "\n************************\n";
}
}

View File

@ -0,0 +1,114 @@
package eu.dnetlib.manager.wf.workflows.graph;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.expression.MapAccessor;
import org.springframework.core.env.Environment;
import org.springframework.expression.ExpressionParser;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.stereotype.Service;
import com.google.common.base.Function;
import com.google.common.collect.Sets;
import eu.dnetlib.errors.WorkflowManagerException;
import eu.dnetlib.manager.wf.model.WorkflowGraph;
import eu.dnetlib.manager.wf.workflows.procs.Env;
import eu.dnetlib.manager.wf.workflows.util.NodeHelper;
@Service
public class GraphLoader {
@Autowired
private NodeHelper nodeHelper;
@Autowired
private Environment environment;
public Graph loadGraph(final WorkflowGraph workflowGraph, final Map<String, String> globalParams)
throws WorkflowManagerException {
final Graph graph = new Graph();
for (final WorkflowGraph.Node node : workflowGraph.getGraph()) {
final String nodeName = node.getName();
final String nodeType = node.getType();
final boolean isStart = node.isStart();
final boolean isJoin = node.isJoin();
final Map<String, Object> params = node.calculateInitialParams(globalParams, environment);
final Map<String, String> envParams = node.findEnvParams();
if (isStart) {
graph.addNode(GraphNode.newStartNode(nodeName, nodeType, params, envParams));
} else if (isJoin) {
graph.addNode(GraphNode.newJoinNode(nodeName, nodeType, params, envParams));
} else {
graph.addNode(GraphNode.newNode(nodeName, nodeType, params, envParams));
}
if (node.getArcs() != null) {
for (final WorkflowGraph.Arc a : node.getArcs()) {
final String to = a.getTo();
final Function<Env, Boolean> condFunction = generateFunction(a.getCondition());
graph.addArc(new Arc(nodeName, to, condFunction));
}
}
graph.addNode(GraphNode.newSuccessNode());
}
checkValidity(graph);
return graph;
}
private Function<Env, Boolean> generateFunction(final String condition) {
return env -> {
final ExpressionParser parser = new SpelExpressionParser();
final StandardEvaluationContext context = new StandardEvaluationContext(env.getAttributes());
context.addPropertyAccessor(new MapAccessor());
return parser.parseExpression(condition).getValue(context, Boolean.class);
};
}
private void checkValidity(final Graph graph) throws WorkflowManagerException {
final Set<String> nodesFromArcs = new HashSet<>();
boolean foundSuccess = false;
boolean foundStart = false;
for (final Arc arc : graph.getArcs()) {
if (StringUtils.isBlank(arc.getFrom()) || StringUtils.isBlank(arc.getFrom())) {
throw new WorkflowManagerException("Invalid arc: missing from e/o to");
}
if (StringUtils.equals(arc.getTo(), GraphNode.SUCCESS_NODE)) {
foundSuccess = true;
}
nodesFromArcs.add(arc.getFrom());
nodesFromArcs.add(arc.getTo());
}
if (!foundSuccess) { throw new WorkflowManagerException("Arc to success not found"); }
final Set<String> diff = Sets.symmetricDifference(graph.nodeNames(), nodesFromArcs);
if (!diff.isEmpty()) { throw new WorkflowManagerException("Missing or invalid nodes in arcs: " + diff); }
for (final GraphNode n : graph.nodes()) {
if (StringUtils.isBlank(n.getName())) { throw new WorkflowManagerException("Invalid node: missing name"); }
if (n.isStart()) {
foundStart = true;
}
if (!this.nodeHelper.isValidType(n.getType())) { throw new WorkflowManagerException("Invalid node type: " + n.getType()); }
}
if (!foundStart) { throw new WorkflowManagerException("Start node not found"); }
}
}

View File

@ -0,0 +1,129 @@
package eu.dnetlib.manager.wf.workflows.graph;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.manager.wf.workflows.procs.Env;
public class GraphNode {
public static final String SUCCESS_NODE = "success";
private final String name;
private final String type;
private final boolean isStart;
private final boolean isJoin;
private final boolean isSucessNode;
private final Map<String, Object> params;
private final Map<String, String> envParams;
private GraphNode(final String name,
final String type,
final boolean isStart,
final boolean isJoin,
final boolean isSuccessNode,
final Map<String, Object> params,
final Map<String, String> envParams) {
this.name = name;
this.type = type;
this.isStart = isStart;
this.isJoin = isJoin;
this.isSucessNode = isSuccessNode;
this.params = params;
this.envParams = envParams;
}
public static GraphNode newNode(final String name,
final String type,
final Map<String, Object> params,
final Map<String, String> envParams) {
return new GraphNode(name, type, false, false, false, params, envParams);
}
public static GraphNode newStartNode(final String name,
final String type,
final Map<String, Object> params,
final Map<String, String> envParams) {
return new GraphNode(name, type, true, false, false, params, envParams);
}
public static GraphNode newJoinNode(final String name,
final String type,
final Map<String, Object> params,
final Map<String, String> envParams) {
return new GraphNode(name, type, false, true, false, params, envParams);
}
public static GraphNode newSuccessNode() {
return new GraphNode(SUCCESS_NODE, null, false, true, true, new HashMap<>(), new HashMap<>());
}
public String getName() {
return this.name;
}
public String getType() {
return this.type;
}
public boolean isStart() {
return this.isStart;
}
public boolean isJoin() {
return this.isJoin;
}
public boolean isSucessNode() {
return this.isSucessNode;
}
@Override
public String toString() {
final StringWriter sw = new StringWriter();
sw.append("[ name: ");
sw.append(this.name);
if (StringUtils.isNotBlank(this.type)) {
sw.append(", type: ");
sw.append(this.type);
}
if (isStart()) {
sw.append(" - isStart");
}
if (isJoin()) {
sw.append(" - isJoin");
}
sw.append(" ]");
return sw.toString();
}
public Map<String, Object> getParams() {
return this.params;
}
public Map<String, String> getEnvParams() {
return this.envParams;
}
public Map<String, Object> resolveParamsWithNoEnv() {
return resolveParams(null);
}
public Map<String, Object> resolveParams(final Env env) {
final Map<String, Object> map = new HashMap<>();
if (this.params != null) {
this.params.forEach((k, v) -> map.put(k, v));
}
if (this.envParams != null && env != null) {
this.envParams.forEach((k, v) -> map.put(k, env.getAttribute(v)));
}
return map;
}
}

View File

@ -0,0 +1,54 @@
package eu.dnetlib.manager.wf.workflows.procs;
import java.util.HashMap;
import java.util.Map;
/**
* Created by michele on 23/11/15.
*/
public class Env {
private final Map<String, Object> attrs;
public Env() {
this.attrs = new HashMap<>();
}
public Env(final Map<String, Object> attrs) {
this.attrs = attrs;
}
public Map<String, Object> getAttributes() {
return attrs;
}
public void clear() {
attrs.clear();
}
public void addAttributes(final Map<String, Object> map) {
if (map != null) {
attrs.putAll(map);
}
}
public void setAttribute(final String name, final Object value) {
attrs.put(name, value);
}
public Object getAttribute(final String name) {
return attrs.get(name);
}
public <T> T getAttribute(final String name, Class<T> clazz) {
return clazz.cast(attrs.get(name));
}
public boolean hasAttribute(final String name) {
return attrs.containsKey(name);
}
public Object removeAttribute(final String name) {
return attrs.remove(name);
}
}

View File

@ -0,0 +1,10 @@
package eu.dnetlib.manager.wf.workflows.procs;
/**
* Created by michele on 24/11/15.
*/
public interface ProcessAware {
void setProcess(WorkflowProcess process);
}

View File

@ -0,0 +1,137 @@
package eu.dnetlib.manager.wf.workflows.procs;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.google.common.base.Throwables;
import eu.dnetlib.manager.history.WorkflowLogger;
import eu.dnetlib.manager.wf.nodes.ProcessNode;
import eu.dnetlib.manager.wf.notification.EmailSender;
import eu.dnetlib.manager.wf.workflows.graph.GraphNode;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
import eu.dnetlib.manager.wf.workflows.util.NodeHelper;
@Service
public class ProcessEngine {
private static final Log log = LogFactory.getLog(ProcessEngine.class);
@Autowired
private NodeHelper nodeHelper;
@Autowired
private EmailSender emailSender;
@Autowired
private WorkflowLogger wfLogger;
public void startProcess(final WorkflowProcess process) {
log.info(process.getGraph());
log.info("Starting workflow: " + process);
final LocalDateTime now = LocalDateTime.now();
process.setStatus(WorkflowProcess.Status.EXECUTING);
process.setStartDate(now);
process.setLastActivityDate(now);
try {
for (final GraphNode node : process.getGraph().startNodes()) {
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, process.getEnv());
final Token token = new Token();
token.getEnv().addAttributes(process.getEnv().getAttributes());
process.getTokens().add(token);
pNode.execute(token, newNodeCallback(process, node));
}
} catch (final Throwable e) {
log.error("WorkflowProcess node instantiation failed", e);
process.setStatus(WorkflowProcess.Status.FAILURE);
}
}
private ExecutionCallback<Token> newNodeCallback(final WorkflowProcess process, final GraphNode node) {
return new ExecutionCallback<Token>() {
@Override
public void onSuccess(final Token t) {
releaseToken(process, node, t);
}
@Override
public void onFail(final Token t) {
completeProcess(process, t);
}
};
}
public void releaseToken(final WorkflowProcess process, final GraphNode oldGraphNode, final Token oldToken) {
process.setLastActivityDate(LocalDateTime.now());
try {
for (final GraphNode node : process.getGraph().nextNodes(oldGraphNode, oldToken.getEnv())) {
if (node.isJoin() || node.isSucessNode()) {
if (!process.getPausedJoinNodeTokens().containsKey(node.getName())) {
process.getPausedJoinNodeTokens().put(node.getName(), new ArrayList<Token>());
}
final List<Token> list = process.getPausedJoinNodeTokens().get(node.getName());
list.add(oldToken);
if (list.size() == process.getGraph().getNumberOfIncomingArcs(node)) {
final Token token = new Token();
token.getEnv().addAttributes(mergeEnvParams(list.toArray(new Token[list.size()])));
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
process.getTokens().add(token);
process.setLastActivityDate(LocalDateTime.now());
if (node.isSucessNode()) {
completeProcess(process, token);
} else {
pNode.execute(token, newNodeCallback(process, node));
}
}
} else {
final Token token = new Token();
token.getEnv().addAttributes(oldToken.getEnv().getAttributes());
final ProcessNode pNode = nodeHelper.newProcessNode(node, process, token.getEnv());
process.getTokens().add(token);
process.setLastActivityDate(LocalDateTime.now());
pNode.execute(token, newNodeCallback(process, node));
}
}
} catch (final Throwable e) {
log.error("WorkflowProcess node instantiation failed", e);
process.setStatus(WorkflowProcess.Status.FAILURE);
process.setError(e.getMessage());
process.setErrorStacktrace(Throwables.getStackTraceAsString(e));
process.setLastActivityDate(LocalDateTime.now());
}
}
private Map<String, Object> mergeEnvParams(final Token... tokens) {
final Map<String, Object> map = new HashMap<>();
Arrays.stream(tokens).forEach(t -> map.putAll(t.getEnv().getAttributes()));
return map;
}
private void completeProcess(final WorkflowProcess process, final Token token) {
token.checkStatus();
process.complete(token);
wfLogger.saveProcessExecution(process.asLog());
emailSender.sendMails(process);
}
}

View File

@ -0,0 +1,59 @@
package eu.dnetlib.manager.wf.workflows.procs;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.errors.WorkflowManagerException;
import eu.dnetlib.is.model.resource.SimpleResource;
import eu.dnetlib.manager.wf.model.WorkflowGraph;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.manager.wf.workflows.graph.Graph;
import eu.dnetlib.manager.wf.workflows.graph.GraphLoader;
import eu.dnetlib.manager.wf.workflows.util.ExecutionCallback;
@Component
public class ProcessFactory {
private static final Log log = LogFactory.getLog(ProcessFactory.class);
private String oldGeneratedId = "";
private final DateTimeFormatter processIdFormatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss_S");
@Autowired
private GraphLoader graphLoader;
public WorkflowProcess newProcess(final SimpleResource wfMetadata,
final WorkflowGraph wfGraph,
final WorkflowInstance wfInstance,
final ExecutionCallback<WorkflowProcess> callback) throws WorkflowManagerException {
final Map<String, String> globalParams = new HashMap<>();
globalParams.putAll(wfInstance.getSystemParams());
globalParams.putAll(wfInstance.getUserParams());
final Graph graph = graphLoader.loadGraph(wfGraph, globalParams);
return new WorkflowProcess(generateProcessId(), wfMetadata, wfInstance, graph, globalParams, callback);
}
private synchronized String generateProcessId() {
String id = "";
do {
id = "wf_" + LocalDateTime.now().format(processIdFormatter);
log.info("Generated processID " + id);
} while (id.equals(oldGeneratedId));
oldGeneratedId = id;
return id;
}
}

View File

@ -0,0 +1,124 @@
package eu.dnetlib.manager.wf.workflows.procs;
import java.time.LocalDateTime;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.PriorityBlockingQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import eu.dnetlib.errors.WorkflowManagerException;
import eu.dnetlib.manager.wf.model.WorkflowInstance;
import eu.dnetlib.manager.wf.workflows.util.WorkflowsConstants;
@Service
public class ProcessRegistry {
private static final Log log = LogFactory.getLog(ProcessRegistry.class);
private final Map<String, WorkflowProcess> procs = new HashMap<>();
private final Map<String, WorkflowProcess> byInstanceId = new HashMap<>();
private final PriorityBlockingQueue<WorkflowProcess> pendingProcs = new PriorityBlockingQueue<>();
@Value("${dnet.wf.registry.size:100}")
private int maxSize;
synchronized public int countRunningWfs() {
int count = 0;
for (final Map.Entry<String, WorkflowProcess> e : this.procs.entrySet()) {
final WorkflowProcess proc = e.getValue();
if (!proc.isTerminated()) {
count++;
}
}
return count;
}
public WorkflowProcess findProcess(final String procId) {
return this.procs.get(procId);
}
public Collection<WorkflowProcess> listProcesses() {
return this.procs.values();
}
public WorkflowProcess findProcsByInstanceId(final String id) {
return this.byInstanceId.get(id);
}
public String registerProcess(final WorkflowProcess process, final WorkflowInstance wfInstance) throws WorkflowManagerException {
if (this.procs.containsValue(process) || this.procs.containsKey(process.getId())) {
log.error("Already registerd process: " + process);
throw new WorkflowManagerException("Already registerd process: " + process);
}
if (this.procs.size() >= this.maxSize) {
removeOldestProcess();
}
this.procs.put(process.getId(), process);
this.byInstanceId.put(wfInstance.getId(), process);
synchronized (this.pendingProcs) {
if (this.pendingProcs.size() > WorkflowsConstants.MAX_PENDING_PROCS_SIZE) {
log.warn("Wf [" + process.getName() + "] not launched, Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE);
throw new WorkflowManagerException("Max number of pending procs reached: " + WorkflowsConstants.MAX_PENDING_PROCS_SIZE);
}
this.pendingProcs.put(process);
log.info("WorkflowProcess [" + process + "] in queue, priority=" + process.getPriority());
}
return process.getId();
}
private void removeOldestProcess() {
LocalDateTime oldDate = LocalDateTime.now();
String oldId = null;
for (final Map.Entry<String, WorkflowProcess> e : this.procs.entrySet()) {
final WorkflowProcess proc = e.getValue();
if (proc.isTerminated()) {
final LocalDateTime date = proc.getLastActivityDate();
if (date.isBefore(oldDate)) {
oldDate = date;
oldId = e.getKey();
}
}
}
if (oldId != null) {
unregisterProcess(oldId);
}
}
public void unregisterProcess(final String procId) {
synchronized (this) {
final WorkflowProcess process = this.procs.remove(procId);
if (process != null) {
final Optional<String> instanceId = this.byInstanceId.entrySet()
.stream()
.filter(e -> e.getValue().getId().equals(process.getId()))
.map(e -> e.getKey())
.findFirst();
if (instanceId.isPresent()) {
this.byInstanceId.remove(instanceId, process);
}
}
}
}
public WorkflowProcess nextProcessToStart() {
synchronized (this.pendingProcs) {
return this.pendingProcs.poll();
}
}
}

View File

@ -0,0 +1,125 @@
package eu.dnetlib.manager.wf.workflows.procs;
import java.time.LocalDateTime;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Throwables;
/**
* Created by michele on 19/11/15.
*/
public class Token {
private final String id;
private final Env env = new Env();
private String progressMessage;
private final LocalDateTime startDate;
private LocalDateTime endDate;
private boolean failed;
private boolean active;
private String error = "";
private String errorStackTrace = "";
public Token() {
this.id = "token-" + UUID.randomUUID();
this.startDate = LocalDateTime.now();
this.failed = false;
this.active = true;
}
public String getId() {
return this.id;
}
public Env getEnv() {
return this.env;
}
public LocalDateTime getStartDate() {
return this.startDate;
}
public LocalDateTime getEndDate() {
return this.endDate;
}
public void setEndDate(final LocalDateTime endDate) {
this.endDate = endDate;
}
public boolean isActive() {
return this.active;
}
public void setActive(final boolean active) {
this.active = active;
}
public boolean isFailed() {
return this.failed;
}
public void setFailed(final boolean failed) {
this.failed = failed;
}
public void release() {
setEndDate(LocalDateTime.now());
setActive(false);
}
public void releaseAsFailed(final Throwable e) {
setEndDate(LocalDateTime.now());
setActive(false);
setFailed(true);
setError(e.getMessage());
setErrorStackTrace(Throwables.getStackTraceAsString(e));
}
public void releaseAsFailed(final String error) {
setEndDate(LocalDateTime.now());
setActive(false);
setFailed(true);
setError(error);
}
public void checkStatus() {
if (isActive()) {
if (StringUtils.isNotBlank(error)) {
releaseAsFailed(error);
} else {
release();
}
}
}
public String getProgressMessage() {
return progressMessage;
}
public void setProgressMessage(final String progressMessage) {
this.progressMessage = progressMessage;
}
public String getError() {
return this.error;
}
public void setError(final String error) {
this.error = error;
}
public String getErrorStackTrace() {
return this.errorStackTrace;
}
public void setErrorStackTrace(final String errorStackTrace) {
this.errorStackTrace = errorStackTrace;
}
}

Some files were not shown because too many files have changed in this diff Show More