diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV1.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV1.java index 972ee657..7d8041e9 100755 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV1.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV1.java @@ -147,8 +147,8 @@ public class DsmApiControllerV1 extends AbstractDsmController { final StopWatch stop = StopWatch.createStarted(); - final Datasource ds = dsmService.getDs(dsId); - final List apis = dsmService.getApis(dsId); + final Datasource ds = dsmService.findDs(dsId); + final List apis = dsmService.findApis(dsId); final List api = apis.stream() .map(DsmMappingUtils::asDetails) .map(a -> a.setEoscDatasourceType(ds.getEoscDatasourceType())) @@ -211,7 +211,7 @@ public class DsmApiControllerV1 extends AbstractDsmController { public void updateDatasource(@RequestBody final DatasourceDetailsUpdate d) throws DsmException, DsmNotFoundException { // initialize with current values from DB - final Datasource ds = dsmService.getDs(d.getId()); + final Datasource ds = dsmService.findDs(d.getId()); if (ds == null) { throw new DsmNotFoundException(String.format("ds '%s' does not exist", d.getId())); } diff --git a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV2.java b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV2.java index f514fd46..68ce8e1f 100644 --- a/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV2.java +++ b/apps/dnet-is-application/src/main/java/eu/dnetlib/dsm/DsmApiControllerV2.java @@ -42,7 +42,7 @@ public class DsmApiControllerV2 extends AbstractDsmController { public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, @RequestParam(required = false) final String typologyFilter) throws Throwable { final StopWatch stop = StopWatch.createStarted(); - final List list = dsmService.getFirstCollectedAfter(fromDate, typologyFilter); + final List list = dsmService.findFirstCollectedAfter(fromDate, typologyFilter); final SimpleResponse rsp = ResponseUtils.simpleResponse(list); return prepareResponse(1, list.size(), stop, rsp); diff --git a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java index 31ce3c2f..f7d72330 100644 --- a/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java +++ b/libs/dnet-is-services/src/main/java/eu/dnetlib/dsm/DsmService.java @@ -154,11 +154,11 @@ public class DsmService { return dsRepository.findAll(spec, PageRequest.of(page, size)); } - public Datasource getDs(final String dsId) throws DsmException { + public Datasource findDs(final String dsId) throws DsmException { return dsRepository.findById(dsId).orElseThrow(() -> new DsmException("Datasource not found. ID: " + dsId)); } - public Datasource getDsByNsPrefix(final String prefix) throws DsmException { + public Datasource findDsByNsPrefix(final String prefix) throws DsmException { return dsRepository.findByNamespaceprefix(prefix).orElseThrow(() -> new DsmException("Datasource not found. NS Prefix: " + prefix)); } @@ -177,7 +177,7 @@ public class DsmService { apiRepository.updateCompatibility(apiId, compliance); } - public List getApis(final String dsId) { + public List findApis(final String dsId) { return apiRepository.findByDatasource(dsId); } @@ -342,7 +342,7 @@ public class DsmService { } } - public List getFirstCollectedAfter(final String fromDate, final String typeFilter) throws DsmException { + public List findFirstCollectedAfter(final String fromDate, final String typeFilter) throws DsmException { try { if (StringUtils.isNotBlank(typeFilter)) { return querySql("first_collected_datasources_fromDate_typology.st.sql", rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate); @@ -522,27 +522,30 @@ public class DsmService { return apiRepository.findById(id).orElseThrow(() -> new DsmException("Api not found. ID: " + id)); } - public void updateUpdateApiCollectionInfo(final String apiId, final String mdId, final long total) { + public Api updateUpdateApiCollectionInfo(final String apiId, final String mdId, final long total) throws DsmException { if (StringUtils.isNotBlank(mdId)) { apiRepository.updateLastCollectionInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total); } else { apiRepository.updateLastCollectionInfo(apiId, null, null, 0); } + return findApi(apiId); } - public void updateUpdateApiAggregationInfo(final String apiId, final String mdId, final long total) { + public Api updateUpdateApiAggregationInfo(final String apiId, final String mdId, final long total) throws DsmException { if (StringUtils.isNotBlank(mdId)) { apiRepository.updateLastAggregationInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total); } else { apiRepository.updateLastAggregationInfo(apiId, null, null, 0); } + return findApi(apiId); } - public void updateUpdateApiDownloadInfo(final String apiId, final String mdId, final long total) { + public Api updateUpdateApiDownloadInfo(final String apiId, final String mdId, final long total) throws DsmException { if (StringUtils.isNotBlank(mdId)) { apiRepository.updateLastDownloadInfo(apiId, mdId, new Timestamp(System.currentTimeMillis()), total); } else { apiRepository.updateLastDownloadInfo(apiId, null, null, 0); } + return findApi(apiId); } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java index 799b8ed3..eddf7a92 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/WorkflowManagerService.java @@ -94,7 +94,7 @@ public class WorkflowManagerService implements Stoppable { } try { - final String dsName = dsmService.getDs(dsId).getOfficialname(); + final String dsName = dsmService.findDs(dsId).getOfficialname(); final WorkflowConfiguration conf = new WorkflowConfiguration(); conf.setId("REPO_HI_" + UUID.randomUUID()); diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/CollectNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/CollectNode.java index 8b5e35ce..8885c943 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/CollectNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/CollectNode.java @@ -5,7 +5,6 @@ import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Autowired; import eu.dnetlib.data.collect.CollectorService; -import eu.dnetlib.dsm.DsmService; import eu.dnetlib.dsm.model.Api; import eu.dnetlib.manager.wf.annotations.WfInputParam; import eu.dnetlib.manager.wf.annotations.WfNode; @@ -15,25 +14,13 @@ import eu.dnetlib.manager.wf.nodes.stream.StreamSupplierNode; public class CollectNode extends StreamSupplierNode { @WfInputParam - private String dsId; - - @WfInputParam - private String apiId; - - @Autowired - private DsmService dsmService; + private Api api; @Autowired private CollectorService collectorService; @Override protected Stream prepareStream() { - final Api api = dsmService.getApis(dsId) - .stream() - .filter(a -> apiId.equals(a.getId())) - .findFirst() - .orElseThrow(); - return collectorService.collect(api); } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/DateRangeCollectNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/DateRangeCollectNode.java index 07b926b0..f140ce74 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/DateRangeCollectNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/DateRangeCollectNode.java @@ -6,7 +6,6 @@ import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Autowired; import eu.dnetlib.data.collect.CollectorService; -import eu.dnetlib.dsm.DsmService; import eu.dnetlib.dsm.model.Api; import eu.dnetlib.manager.wf.annotations.WfInputParam; import eu.dnetlib.manager.wf.annotations.WfNode; @@ -16,10 +15,7 @@ import eu.dnetlib.manager.wf.nodes.stream.StreamSupplierNode; public class DateRangeCollectNode extends StreamSupplierNode { @WfInputParam - private String dsId; - - @WfInputParam - private String apiId; + private Api api; @WfInputParam private LocalDateTime from; @@ -27,22 +23,12 @@ public class DateRangeCollectNode extends StreamSupplierNode { @WfInputParam private LocalDateTime until; - @Autowired - private DsmService dsmService; - @Autowired private CollectorService collectorService; @Override protected Stream prepareStream() { - final Api api = dsmService.getApis(dsId) - .stream() - .filter(a -> apiId.equals(a.getId())) - .findFirst() - .orElseThrow(); - return collectorService.collect(api, from, until); - } @Override diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/FindDateRangeForIncrementalHarvestingNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/FindDateRangeForIncrementalHarvestingNode.java index f28b8d87..9b89d4de 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/FindDateRangeForIncrementalHarvestingNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/FindDateRangeForIncrementalHarvestingNode.java @@ -2,9 +2,7 @@ package eu.dnetlib.manager.wf.nodes.aggregation; import java.time.LocalDateTime; -import org.springframework.beans.factory.annotation.Autowired; - -import eu.dnetlib.dsm.DsmService; +import eu.dnetlib.dsm.model.Api; import eu.dnetlib.manager.wf.annotations.WfInputParam; import eu.dnetlib.manager.wf.annotations.WfNode; import eu.dnetlib.manager.wf.annotations.WfOutputParam; @@ -14,10 +12,7 @@ import eu.dnetlib.manager.wf.nodes.AbstractJobNode; public class FindDateRangeForIncrementalHarvestingNode extends AbstractJobNode { @WfInputParam - private String dsId; - - @WfInputParam - private String apiId; + private Api api; @WfInputParam private LocalDateTime overrideFromDate; @@ -31,9 +26,6 @@ public class FindDateRangeForIncrementalHarvestingNode extends AbstractJobNode { @WfOutputParam private LocalDateTime untilDate; - @Autowired - private DsmService dsmService; - @Override protected void execute() throws Exception { fromDate = overrideFromDate != null ? overrideFromDate : findLastCollDate(); @@ -41,12 +33,7 @@ public class FindDateRangeForIncrementalHarvestingNode extends AbstractJobNode { } private LocalDateTime findLastCollDate() { - return dsmService.getApis(dsId) - .stream() - .filter(a -> apiId.equals(a.getId())) - .findFirst() - .map(a -> a.getLastCollectionDate().toLocalDateTime()) - .orElse(null); + return api.getLastCollectionDate() != null ? api.getLastCollectionDate().toLocalDateTime() : null; } } diff --git a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/OpenaireMdBuilderNode.java b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/OpenaireMdBuilderNode.java index 374434f0..1582f911 100644 --- a/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/OpenaireMdBuilderNode.java +++ b/libs/dnet-wf-service/src/main/java/eu/dnetlib/manager/wf/nodes/aggregation/OpenaireMdBuilderNode.java @@ -8,6 +8,8 @@ import eu.dnetlib.data.mdstore.model.records.MetadataRecord; import eu.dnetlib.data.mdstore.model.records.MetadataRecordImpl; import eu.dnetlib.data.mdstore.model.records.Provenance; import eu.dnetlib.dsm.DsmService; +import eu.dnetlib.dsm.model.Api; +import eu.dnetlib.dsm.model.Datasource; import eu.dnetlib.manager.wf.annotations.WfInputParam; import eu.dnetlib.manager.wf.annotations.WfNode; import eu.dnetlib.manager.wf.nodes.stream.StreamMapperNode; @@ -16,13 +18,9 @@ import eu.dnetlib.manager.wf.nodes.stream.StreamMapperNode; public class OpenaireMdBuilderNode extends StreamMapperNode { @WfInputParam - private String dsId; + private Datasource ds; @WfInputParam - private String dsPrefix; - @WfInputParam - private String dsName; - @WfInputParam - private String apiId; + private Api api; @WfInputParam private boolean inferred; @WfInputParam @@ -47,17 +45,12 @@ public class OpenaireMdBuilderNode extends StreamMapperNode apiId.equals(a.getId())) - .findFirst() - .map(a -> a.getMetadataIdentifierPath()) - .orElseThrow(); + xpath = api.getMetadataIdentifierPath(); provenance = new Provenance(); - provenance.setDatasourceId(dsId); - provenance.setDatasourceName(dsName); - provenance.setNsPrefix(dsPrefix); + provenance.setDatasourceId(ds.getId()); + provenance.setDatasourceName(ds.getOfficialname()); + provenance.setNsPrefix(ds.getNamespaceprefix()); now = System.currentTimeMillis(); } @@ -73,7 +66,7 @@ public class OpenaireMdBuilderNode extends StreamMapperNode apiId.equals(a.getId())) - .map(a -> ObjectUtils.firstNonNull(a.getCompatibilityOverride(), a.getCompatibility())) - .findFirst() - .orElseThrow(() -> new Exception("Missing api: " + apiId)); + final String type = ds.getEoscDatasourceType(); + final String compatibility = ObjectUtils.firstNonNull(api.getCompatibilityOverride(), api.getCompatibility()); if (StringUtils.isNotBlank(expectedEoscDatasourceTypes)) { Arrays.stream(expectedEoscDatasourceTypes.split(",")) diff --git a/libs/dnet-wf-service/src/main/resources/wf_templates/aggregation.json b/libs/dnet-wf-service/src/main/resources/wf_templates/aggregation.json index 1f10a194..bf75d85e 100644 --- a/libs/dnet-wf-service/src/main/resources/wf_templates/aggregation.json +++ b/libs/dnet-wf-service/src/main/resources/wf_templates/aggregation.json @@ -2,19 +2,11 @@ "parameters": [ { "name":"dsId", - "description":"Datasource ID" - }, - { - "name":"dsPrefix", - "description":"Datasource Prefix" - }, - { - "name":"dsName", - "description":"Datasource Name" + "description":"the Datasource ID" }, { "name":"apiId", - "description":"Datasource API" + "description":"the Api ID" }, { "name":"nativeMdStoreId", diff --git a/libs/dnet-wf-service/src/main/resources/wf_templates/collect.json b/libs/dnet-wf-service/src/main/resources/wf_templates/collect.json index ba791dd7..f0c6e5b0 100644 --- a/libs/dnet-wf-service/src/main/resources/wf_templates/collect.json +++ b/libs/dnet-wf-service/src/main/resources/wf_templates/collect.json @@ -2,19 +2,11 @@ "parameters":[ { "name":"dsId", - "description":"Datasource ID" - }, - { - "name":"dsPrefix", - "description":"Datasource Prefix" - }, - { - "name":"dsName", - "description":"Datasource Name" + "description":"the Datasource ID" }, { "name":"apiId", - "description":"Datasource API" + "description":"the Api ID" }, { "name":"mode", @@ -39,9 +31,38 @@ } ], "graph":[ + { + "name":"LOAD_DS_INFO", + "type":"loadDatasourceInfo", + "start": true, + "input":[ + { + "name":"dsId", + "ref":"dsId" + }, + { + "name":"apiId", + "ref":"apiId" + } + ], + "output":[ + { + "name":"ds", + "env":"ds" + }, + { + "name":"api", + "env":"api" + } + ], + "arcs":[ + { + "to":"SELECT_MODE" + } + ] + }, { "name":"SELECT_MODE", - "start":true, "arcs":[ { "to":"COLLECT_REFRESH", @@ -58,12 +79,8 @@ "type":"collect", "input":[ { - "name":"dsId", - "ref":"dsId" - }, - { - "name":"apiId", - "ref":"apiId" + "name":"api", + "env":"api" } ], "output":[ @@ -87,12 +104,8 @@ "type":"findDateRangeForIncrementalHarvesting", "input":[ { - "name":"dsId", - "ref":"dsId" - }, - { - "name":"apiId", - "ref":"apiId" + "name":"api", + "env":"api" }, { "name":"from", @@ -124,12 +137,8 @@ "type":"dateRangeCollect", "input":[ { - "name":"dsId", - "ref":"dsId" - }, - { - "name":"apiId", - "ref":"apiId" + "name":"api", + "env":"api" }, { "name":"fromDate", @@ -161,24 +170,12 @@ "type":"openaireMdBuilder", "input":[ { - "name":"dsId", - "ref":"dsId" + "name":"ds", + "env":"ds" }, { - "name":"dsPrefix", - "ref":"dsPrefix" - }, - { - "name":"dsName", - "ref":"dsName" - }, - { - "name":"apiId", - "ref":"apiId" - }, - { - "name": "xpathForRecordId", - "env" : "xpathForRecordId" + "name":"api", + "env":"api" }, { "name":"inputStream", @@ -255,12 +252,8 @@ "type":"updateApiExtraFields", "input":[ { - "name":"dsId", - "ref":"dsId" - }, - { - "name":"apiId", - "ref":"apiId" + "name":"api", + "env":"api" }, { "name":"infoType", @@ -274,6 +267,12 @@ "name":"total", "env":"total" } + ], + "output":[ + { + "name":"api", + "env": "api" + } ] } ] diff --git a/libs/dnet-wf-service/src/main/resources/wf_templates/transform.json b/libs/dnet-wf-service/src/main/resources/wf_templates/transform.json index 27b765b1..f87973ce 100644 --- a/libs/dnet-wf-service/src/main/resources/wf_templates/transform.json +++ b/libs/dnet-wf-service/src/main/resources/wf_templates/transform.json @@ -2,19 +2,11 @@ "parameters":[ { "name":"dsId", - "description":"Datasource ID" - }, - { - "name":"dsPrefix", - "description":"Datasource Prefix" - }, - { - "name":"dsName", - "description":"Datasource Name" + "description":"the Datasource ID" }, { "name":"apiId", - "description":"Datasource API" + "description":"the Api ID" }, { "name":"inputMdStoreId", @@ -26,10 +18,39 @@ } ], "graph":[ + { + "name":"LOAD_DS_INFO", + "type":"loadDatasourceInfo", + "start": true, + "input":[ + { + "name":"dsId", + "ref":"dsId" + }, + { + "name":"apiId", + "ref":"apiId" + } + ], + "output":[ + { + "name":"ds", + "env":"ds" + }, + { + "name":"api", + "env":"api" + } + ], + "arcs":[ + { + "to":"FETCH_MDSTORE" + } + ] + }, { "name":"FETCH_MDSTORE", "type":"fetchMdStore", - "start": true, "input":[ { "name":"mdId", @@ -107,12 +128,8 @@ "type":"updateApiExtraFields", "input":[ { - "name":"dsId", - "ref":"dsId" - }, - { - "name":"apiId", - "ref":"apiId" + "name":"api", + "env":"api" }, { "name":"infoType",