preparation of aggr nodes

This commit is contained in:
Michele Artini 2023-04-27 08:31:25 +02:00
parent 73bbb6d6b0
commit 3737f5fa6c
4 changed files with 77 additions and 7 deletions

View File

@ -0,0 +1,32 @@
package eu.dnetlib.manager.wf.nodes.aggregation;
import java.time.LocalDateTime;
import java.util.stream.Stream;
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
import eu.dnetlib.manager.wf.annotations.WfInputParam;
import eu.dnetlib.manager.wf.annotations.WfNode;
import eu.dnetlib.manager.wf.nodes.stream.StreamSupplierNode;
@WfNode("collect")
public class CollectNode extends StreamSupplierNode<MetadataRecord> {
@WfInputParam
private String dsId;
@WfInputParam
private String apiId;
@WfInputParam
private LocalDateTime from;
@WfInputParam
private LocalDateTime until;
@Override
protected Stream<MetadataRecord> prepareStream() {
// TODO Auto-generated method stub
return null;
}
}

View File

@ -7,11 +7,11 @@ import eu.dnetlib.manager.wf.annotations.WfInputParam;
import eu.dnetlib.manager.wf.annotations.WfNode;
import eu.dnetlib.manager.wf.nodes.stream.StreamSupplierNode;
@WfNode("oai_collect")
public class CollectOAINode extends StreamSupplierNode<MetadataRecord> {
@WfNode("collect")
public class DateRangeCollectNode extends StreamSupplierNode<MetadataRecord> {
@WfInputParam
private String datasourceID;
private String dsId;
@WfInputParam
private String apiId;

View File

@ -0,0 +1,38 @@
package eu.dnetlib.manager.wf.nodes.aggregation;
import java.time.LocalDateTime;
import eu.dnetlib.manager.wf.annotations.WfInputParam;
import eu.dnetlib.manager.wf.annotations.WfNode;
import eu.dnetlib.manager.wf.annotations.WfOutputParam;
import eu.dnetlib.manager.wf.nodes.AbstractJobNode;
@WfNode("findDateRangeForIncrementalHarvesting")
public class FindDateRangeForIncrementalHarvestingNode extends AbstractJobNode {
public FindDateRangeForIncrementalHarvestingNode() {
super(false);
}
@WfInputParam
private LocalDateTime overrideFromDate;
@WfInputParam
private LocalDateTime overrideUntilDate;
@WfOutputParam
private LocalDateTime fromDate;
@WfOutputParam
private LocalDateTime untilDate;
@Override
protected void execute() {
fromDate = overrideFromDate != null ? overrideFromDate : findLastAggrDate();
untilDate = overrideUntilDate != null ? overrideUntilDate : LocalDateTime.now();
}
private LocalDateTime findLastAggrDate() {
// TODO Auto-generated method stub
return null;
}
}

View File

@ -55,7 +55,7 @@
},
{
"name":"COLLECT_REFRESH",
"type":"Collect",
"type":"collect",
"input":[
{
"name":"dsId",
@ -80,7 +80,7 @@
},
{
"name":"PREPARE_INCREMENTAL",
"type":"FindDateRangeForIncrementalHarvesting",
"type":"findDateRangeForIncrementalHarvesting",
"input":[
{
"name":"from",
@ -109,7 +109,7 @@
},
{
"name":"COLLECT_INCREMENTAL",
"type":"DateRangeCollect",
"type":"dateRangeCollect",
"input":[
{
"name":"dsId",
@ -142,7 +142,7 @@
},
{
"name":"MD_BUILDER",
"type":"OpenaireMdBuilder",
"type":"openaireMdBuilder",
"input":[
{
"name":"dsId",