michele.artini
/
dnet-docker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

start repo-hi api

This commit is contained in:
Michele Artini 2023-11-30 14:39:55 +01:00
parent 892a6adf8e
commit f4ef6847c5
15 changed files with 202 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
dnet-app/apps/dnet-wf-aggregation-postgres/src/main/java/eu/dnetlib/wfs/controller/WfExecutorApiController.java
View File

@ -1,31 +1,10 @@
package eu.dnetlib.wfs.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.common.controller.DnetRestController;
import eu.dnetlib.wfs.procs.WorkflowProcess;
import eu.dnetlib.wfs.service.WfExecutorService;
@RestController
@RequestMapping("/api/wfs")
public class WfExecutorRestController extends DnetRestController {
@Autowired
private WfExecutorService wfExecutorService;
@GetMapping("/process/{id}")
public WorkflowProcess findProcess(@PathVariable final String id) throws Exception {
return wfExecutorService.findProcess(id);
}
@DeleteMapping("/process/{id}")
public void killProcess(@PathVariable final String id) throws Exception {
wfExecutorService.killProcess(id);
}
public class WfExecutorApiController extends AbstractWfExecutorApiController {
}

8
dnet-app/apps/dnet-wf-manager/src/main/java/eu/dnetlib/wfs/manager/controller/ApiController.java
View File

@ -18,9 +18,11 @@ import eu.dnetlib.domain.common.KeyValue;
import eu.dnetlib.domain.wfs.WfConfiguration;
import eu.dnetlib.domain.wfs.WfJournalEntry;
import eu.dnetlib.domain.wfs.WfRepoHiDesc;
import eu.dnetlib.domain.wfs.WfRepoHiParams;
import eu.dnetlib.domain.wfs.WfSection;
import eu.dnetlib.domain.wfs.WfSubscription;
import eu.dnetlib.domain.wfs.WfTemplate;
import eu.dnetlib.domain.wfs.WorkflowsConstants;
import eu.dnetlib.wfs.manager.service.WorkflowManagerService;
@RestController
@ -108,6 +110,12 @@ public class ApiController extends DnetRestController {
return wfManagerService.listRepoHis();
}
@PostMapping("/repo-hi/{id}/start")
public WfJournalEntry startRepoHi(@PathVariable final String id, @RequestBody final WfRepoHiParams params) {
return wfManagerService
.prepareNewJob(id, WorkflowsConstants.REPO_HI_JOB, WorkflowsConstants.REPO_HI_JOB, params.getDsId(), params.getDsName(), params.getApiId());
}
@GetMapping("/template/{id}")
public WfTemplate findWfTemplate(@PathVariable final String id) {
return wfManagerService.findWfTemplate(id);

53
dnet-app/apps/dnet-wf-manager/src/main/java/eu/dnetlib/wfs/manager/service/WorkflowManagerService.java
View File

@ -3,8 +3,8 @@ package eu.dnetlib.wfs.manager.service;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.TimeZone;
import java.util.UUID;
@ -132,6 +132,35 @@ public class WorkflowManagerService {
return wfJournalEntryRepository.save(job);
}
@Transactional
public WfJournalEntry prepareNewJob(final String wfTemplateId,
final String name,
final String family,
final String dsId,
final String dsName,
final String apiId) {
final WfJournalEntry job = new WfJournalEntry();
job.setProcessId(WfProcessUtils.generateProcessId());
job.setWfConfigurationId(null);
job.setName(name);
job.setFamily(family);
job.setWfTemplateId(wfTemplateId);
job.setDsId(dsId);
job.setDsName(dsName);
job.setDsApi(apiId);
job.setStatus(JobStatus.pending);
job.setDetails(new LinkedHashMap<>());
job.setStartDate(null);
job.setEndDate(null);
job.setWfExecutor(null);
return wfJournalEntryRepository.save(job);
}
public List<WfJournalEntry> findHistoryForDsId(final String dsId) {
return wfJournalEntryRepository.findByDsIdOrderByEndDateDesc(dsId, PageRequest.of(0, MAX_HISTORY_SIZE)).getContent();
}
@ -175,9 +204,25 @@ public class WorkflowManagerService {
repohi.setParameters(tmpl.getParameters());
repohi.setInfo(tmpl.getInfo());
// TODO (MEDIUM PRIORITY)
repohi.setExpectedEoscDsTypes(new HashSet<>());
repohi.setExpectedCompliances(new HashSet<>());
repohi.setExpectedEoscDsTypes(new LinkedHashSet<>());
repohi.setExpectedCompliances(new LinkedHashSet<>());
tmpl.getParameters().forEach(p -> {
if ("expectedEoscDsTypes".equalsIgnoreCase(p.getName())) {
for (final String s : StringUtils.split(p.getDefaultValue(), ",")) {
if (StringUtils.isNotBlank(s)) {
repohi.getExpectedEoscDsTypes().add(s.trim());
}
}
}
if ("expectedCompliances".equalsIgnoreCase(p.getName())) {
for (final String s : StringUtils.split(p.getDefaultValue(), ",")) {
if (StringUtils.isNotBlank(s)) {
repohi.getExpectedCompliances().add(s.trim());
}
}
}
});
return repohi;
})

72
dnet-app/apps/dnet-wf-manager/src/main/resources/wf_templates/simple_repo_hi.json
View File

@ -3,12 +3,12 @@
{
"name":"wfId",
"description":"the workflow to be configured",
"defaultValue": "wf-aggr-dc2oaf"
"defaultValue": "wf-aggr-dc2native"
},
{
"name":"destroyWfId",
"description":"the workflow to remove the configuration",
"defaultValue": "wf-destroy-dc2oaf"
"defaultValue": "wf-destroy-dc2native"
},
{
"name":"dsId",
@ -19,12 +19,14 @@
"description":"the Api ID"
},
{
"name":"expectedEoscDatasourceTypes",
"description":"the expected ds types (comma separated)"
"name":"expectedEoscDsTypes",
"description":"the expected ds types (comma separated)",
"defaultValue":"Repository"
},
{
"name":"expectedCompatibilities",
"description":"the expected compatibilities (comma separated)"
"name":"expectedCompliances",
"description":"the expected compatibilities (comma separated)",
"defaultValue":"UNKNOWN,DRIVER,OPENAIRE,ISTI"
}
],
"graph":[
@ -63,12 +65,12 @@
"type":"verifyDatasource",
"input":[
{
"name":"expectedEoscDatasourceTypes",
"ref":"expectedEoscDatasourceTypes"
"name":"expectedEoscDsTypes",
"ref":"expectedEoscDsTypes"
},
{
"name":"expectedCompatibilities",
"ref":"expectedCompatibilities"
"name":"expectedCompliances",
"ref":"expectedCompliances"
},
{
"name":"ds",
@ -80,18 +82,13 @@
}
],
"output":[
{
"name":"outputStream",
"env":"origStream"
}
],
"output":[],
"arcs":[
{
"to":"CREATE_DC"
},
{
"to":"CREATE_OAF"
"to":"CREATE_NATIVE"
}
]
},
@ -105,12 +102,16 @@
},
{
"name":"layout",
"value":"native"
"value":"original"
},
{
"name":"interpretation",
"value":"store"
},
{
"name":"backend",
"value":"SQL_DB"
},
{
"name":"ds",
"env":"ds"
@ -133,12 +134,12 @@
]
},
{
"name":"CREATE_OAF",
"name":"CREATE_NATIVE",
"type":"createMdStore",
"input":[
{
"name":"format",
"value":"OAF"
"value":"native"
},
{
"name":"layout",
@ -148,6 +149,10 @@
"name":"interpretation",
"value":"store"
},
{
"name":"backend",
"value":"SQL_DB"
},
{
"name":"ds",
"env":"ds"
@ -160,7 +165,7 @@
"output":[
{
"name":"mdId",
"env":"mdId_oaf"
"env":"mdId_cleaned"
}
],
"arcs":[
@ -197,7 +202,7 @@
},
{
"name":"cleanedMdStoreId",
"env":"mdId_oaf"
"env":"mdId_cleaned"
}
],
"output":[
@ -209,26 +214,3 @@
}
]
}

28
dnet-app/frontends/is/src/app/common/is.model.ts
View File

@ -45,19 +45,6 @@ export interface Protocol {
params: ProtocolParam[]
}
export interface WfHistoryEntry {
processId: string,
name: string,
family: string,
status: string,
startDate: string,
endDate: string,
dsId?: string,
dsName?: string,
dsApi?: string,
details: Map<string, string>
}
export interface SimpleResource {
id: string,
name: string,
@ -248,11 +235,20 @@ export interface WfParam {
required: boolean
}
export interface WfSubscription {
// TODO
export interface WfHistoryEntry {
processId: string,
name: string,
family: string,
status: string,
startDate: string,
endDate: string,
dsId?: string,
dsName?: string,
dsApi?: string,
details: Map<string, string>
}
export interface WfProcessStatus {
export interface WfSubscription {
// TODO
}

6
dnet-app/frontends/is/src/app/dsm/dsm-api.component.html
View File

@ -31,7 +31,11 @@
</td>
</tr>
<tr>
<th align="left" width="20%">Email</th>
<th align="left">EOSC Datasource Type</th>
<td>{{ds.eoscDatasourceType}}</td>
</tr>
<tr>
<th align="left">Email</th>
<td width="80%">{{ds.contactemail}}</td>
</tr>
<tr>

7
dnet-app/frontends/is/src/app/dsm/dsm.client.ts
View File

@ -1,5 +1,5 @@
import { Injectable } from '@angular/core';
import { Page, DsmConf, KeyValue, BrowseTerm, SimpleDatasource, WfRepoHi, WfTemplate, WfConf } from '../common/is.model';
import { Page, DsmConf, KeyValue, BrowseTerm, SimpleDatasource, WfRepoHi, WfConf } from '../common/is.model';
import { ISClient } from '../common/is.client';
@Injectable({
@ -45,8 +45,9 @@ export class DsmClient extends ISClient {
this.httpGet<WfRepoHi>('/proxy/byType/wf_manager/api/repo-his', onSuccess);
}
dsmRepoHiWf(wfId: string, onSuccess: Function) {
alert('TODO');
dsmRepoHiWf(wfId: string, dsId: string, dsName: string, apiId: string, onSuccess: Function) {
let params = { 'dsId': dsId, 'dsName': dsName, 'apiId': apiId };
this.httpPost('/proxy/byType/wf_manager/api/repo-hi/' + encodeURIComponent(wfId) + '/start', params, onSuccess);
}
dsmRepoByeWf(wfId: string, onSuccess: Function) {

7
dnet-app/frontends/is/src/app/dsm/dsm.component.ts
View File

@ -1,5 +1,5 @@
import { Component, Inject, Injectable, OnInit, ViewChild } from '@angular/core';
import { Page, BrowseTerm, SimpleDatasource, KeyValue, DsmConf, ProtocolParam, Api, ApiInsert, WfRepoHi, WfTemplate, WfConf } from '../common/is.model';
import { Page, BrowseTerm, SimpleDatasource, KeyValue, DsmConf, ProtocolParam, Api, ApiInsert, WfRepoHi, WfTemplate, WfConf, WfHistoryEntry } from '../common/is.model';
import { ActivatedRoute, Params } from '@angular/router';
import { MatDialog, MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
import { MatTableDataSource } from '@angular/material/table';
@ -354,7 +354,10 @@ export class DsmAddWorkflowDialog {
}
startRepoHiWf(wfId: string): void {
alert('TODO REPO HI');
this.client.dsmRepoHiWf(wfId, this.ds.id, this.ds.name, this.api.id, (data: WfHistoryEntry) => {
alert(data);
this.dialogRef.close(1);
});
}
showGraphModal(wf: WfRepoHi): void {

2
dnet-app/frontends/is/src/app/wf-confs/wf-common.component.ts
View File

@ -5,7 +5,7 @@ import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dial
import { MatSelectChange } from '@angular/material/select';
import { MatSnackBar } from '@angular/material/snack-bar';
import { ActivatedRoute, Router } from '@angular/router';
import { KeyValue, SimpleResource, WfConf, WfHistoryEntry, WfParam, WfProcessStatus, WfSection, WfTemplateDesc } from '../common/is.model';
import { KeyValue, SimpleResource, WfConf, WfHistoryEntry, WfParam, WfSection, WfTemplateDesc } from '../common/is.model';
import { ResMetadataDialog } from '../resources/resources.component';
import { MatTableDataSource } from '@angular/material/table';
import { WfHistoryDialog } from '../wf-history/wf-history.component';

8
dnet-app/frontends/is/src/app/wf-confs/wf-confs.client.ts
View File

@ -1,6 +1,6 @@
import { Injectable } from '@angular/core';
import { HttpHeaders } from '@angular/common/http';
import { WfHistoryEntry, SimpleResource, KeyValue, WfConf, WfSubscription, WfProcessStatus, WfSection } from '../common/is.model';
import { WfHistoryEntry, SimpleResource, KeyValue, WfConf, WfSubscription, WfSection } from '../common/is.model';
import { FormGroup } from '@angular/forms';
import { ISClient } from '../common/is.client';
@ -34,15 +34,15 @@ export class WfConfsClient extends ISClient {
}
startWfConfiguration(id: string, onSuccess: Function): void {
this.httpGet<WfProcessStatus>('/proxy/byType/wf-manager/conf/' + encodeURIComponent(id) + '/start', onSuccess);
this.httpGet<WfHistoryEntry>('/proxy/byType/wf-manager/conf/' + encodeURIComponent(id) + '/start', onSuccess);
}
startDestroyWfConfiguration(id: string, onSuccess: Function): void {
this.httpGet<WfProcessStatus>('/proxy/byType/wf-manager/conf/' + encodeURIComponent(id) + '/destroy', onSuccess);
this.httpGet<WfHistoryEntry>('/proxy/byType/wf-manager/conf/' + encodeURIComponent(id) + '/destroy', onSuccess);
}
findProcess(id: string, onSuccess: Function): void {
this.httpGet<WfProcessStatus>('/proxy/byType/wf-manager/process/' + encodeURIComponent(id), onSuccess);
this.httpGet<WfHistoryEntry>('/proxy/byType/wf-manager/process/' + encodeURIComponent(id), onSuccess);
}
killProcess(id: string, onSuccess: Function): void {

4
dnet-app/frontends/is/src/styles.css
View File

@ -140,7 +140,7 @@ a:not([href]):hover {
/* Mermaid Graph */
.graphStartNode>rect {
fill: white !important;
fill: #ccccff !important;
stroke: #336699 !important;
stroke-width: 3px !important;
}
@ -153,7 +153,7 @@ a:not([href]):hover {
}
.graphSimpleNode>rect {
fill: #ccccff !important;
fill: white !important;
stroke: #336699 !important;
stroke-width: 2px !important;
}

37
dnet-app/libs/dnet-domain/src/main/java/eu/dnetlib/domain/wfs/WfRepoHiParams.java Normal file
View File

@ -0,0 +1,37 @@
package eu.dnetlib.domain.wfs;
import java.io.Serializable;
public class WfRepoHiParams implements Serializable {
private static final long serialVersionUID = 4130982720455226772L;
private String dsId;
private String dsName;
private String apiId;
public String getDsId() {
return dsId;
}
public void setDsId(final String dsId) {
this.dsId = dsId;
}
public String getDsName() {
return dsName;
}
public void setDsName(final String dsName) {
this.dsName = dsName;
}
public String getApiId() {
return apiId;
}
public void setApiId(final String apiId) {
this.apiId = apiId;
}
}

37
dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/controller/AbstractWfExecutorApiController.java Normal file
View File

@ -0,0 +1,37 @@
package eu.dnetlib.wfs.controller;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import eu.dnetlib.common.controller.DnetRestController;
import eu.dnetlib.wfs.graph.GraphLoader;
import eu.dnetlib.wfs.procs.WorkflowProcess;
import eu.dnetlib.wfs.service.WfExecutorService;
public abstract class AbstractWfExecutorApiController extends DnetRestController {
@Autowired
private WfExecutorService wfExecutorService;
@Autowired
private GraphLoader graphLoader;
@GetMapping("/process/{id}")
public WorkflowProcess findProcess(@PathVariable final String id) throws Exception {
return wfExecutorService.findProcess(id);
}
@DeleteMapping("/process/{id}")
public void killProcess(@PathVariable final String id) throws Exception {
wfExecutorService.killProcess(id);
}
@GetMapping("/node-types")
public List<String> listAvailableNodes() {
return graphLoader.getValidTypes().stream().sorted().toList();
}
}

4
dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/graph/GraphLoader.java
View File

@ -145,4 +145,8 @@ public class GraphLoader {
if (!foundStart) { throw new WorkflowManagerException("Start node not found"); }
}
public Set<String> getValidTypes() {
return validTypes;
}
}

12
dnet-app/libs/dnet-wf-executor-common/src/main/java/eu/dnetlib/wfs/nodes/ds/VerifyDatasourceNode.java
View File

@ -15,10 +15,10 @@ import eu.dnetlib.wfs.nodes.AbstractJobNode;
public class VerifyDatasourceNode extends AbstractJobNode {
@WfInputParam
private String expectedEoscDatasourceTypes;
private String expectedEoscDsTypes;
@WfInputParam
private String expectedCompatibilities;
private String expectedCompliances;
@WfInputParam
private Datasource ds;
@ -31,16 +31,16 @@ public class VerifyDatasourceNode extends AbstractJobNode {
final String type = ds.getEoscDatasourceType();
final String compatibility = ObjectUtils.firstNonNull(api.getCompatibilityOverride(), api.getCompatibility());
if (StringUtils.isNotBlank(expectedEoscDatasourceTypes)) {
Arrays.stream(expectedEoscDatasourceTypes.split(","))
if (StringUtils.isNotBlank(expectedEoscDsTypes)) {
Arrays.stream(expectedEoscDsTypes.split(","))
.filter(StringUtils::isNotBlank)
.filter(s -> s.equalsIgnoreCase(type))
.findFirst()
.orElseThrow(() -> new Exception("Invalid type: " + type));
}
if (StringUtils.isNotBlank(expectedCompatibilities)) {
Arrays.stream(expectedCompatibilities.split(","))
if (StringUtils.isNotBlank(expectedCompliances)) {
Arrays.stream(expectedCompliances.split(","))
.filter(StringUtils::isNotBlank)
.filter(s -> s.equalsIgnoreCase(compatibility))
.findFirst()