gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Task/17692

This commit is contained in:
Fabio Sinibaldi 2020-01-15 17:50:53 +01:00
parent dab91aa8a4
commit 857ab34e64
8 changed files with 61 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DMAlgorithmsInfoCollector.java
View File

@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DMAlgorithmsInfoCollector implements DataCollector<InternalAlgorithmDescriptor> {
private static final Pattern p = Pattern.compile("\\{Published by (.*)\\((.*\\..*\\)).*\\}$");
private static final Pattern p = Pattern.compile("\\{Published by (.*)\\((.*(\\.)?.*\\)).*\\}$");
private Map<String,String> env=null;

2
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/DataMinerPlugin.java
View File

@ -69,7 +69,7 @@ public class DataMinerPlugin implements CollectorPlugin<InternalAlgorithmDescrip
InputStream is = getClass().getResourceAsStream("profile.xml");
java.util.Scanner scanner = new java.util.Scanner(is).useDelimiter("\\A");
String json = scanner.hasNext() ? scanner.next() : "";
GCatModel.setProfile(json);
GCatModel.setStaticProfile(json);
DataMinerCollectorProperties.init();

4
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/GCATTransformer.java
View File

@ -13,8 +13,10 @@ public class GCATTransformer implements DataTransformer<GCatModel,InternalAlgori
@Override
public Set<GCatModel> transform(Collection<InternalAlgorithmDescriptor> collectedData) {
HashSet<GCatModel> toReturn=new HashSet<>();
boolean specifyProfile=true;
for(InternalAlgorithmDescriptor desc:collectedData) {
toReturn.add(desc.asCKANModel());
toReturn.add(desc.asCKANModel(specifyProfile));
specifyProfile=false; //only once
}
return toReturn;
}

6
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/InternalAlgorithmDescriptor.java
View File

@ -45,8 +45,10 @@ public class InternalAlgorithmDescriptor implements CustomData {
private List<String> tags=new ArrayList<>();
public GCatModel asCKANModel() {
return new GCatModel(this);
public GCatModel asCKANModel(boolean defineProfile) {
GCatModel toReturn=new GCatModel(this);
if(!defineProfile) toReturn.setProfile(null);
return toReturn;
}

71
DataMinerAlgorithmsCrawler/src/main/java/org/gcube/data/publishing/gCatFeeder/collectors/dm/model/ckan/GCatModel.java
View File

@ -1,6 +1,7 @@
package org.gcube.data.publishing.gCatFeeder.collectors.dm.model.ckan;
import java.io.ByteArrayOutputStream;
import java.net.URL;
import java.util.ArrayList;
import org.gcube.data.publishing.gCatFeeder.collectors.dm.DataMinerCollectorProperties;
@ -16,29 +17,31 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@NoArgsConstructor
@Getter
@Setter
@Slf4j
public class GCatModel implements CatalogueFormatData {
private static ObjectMapper mapper=new ObjectMapper();
private static String profileXML=null;
String profileID=DataMinerCollectorProperties.getProperty(DataMinerCollectorProperties.CKAN_RESOURCE_TYPE);
public static void setProfile(String toSet) {
public static void setStaticProfile(String toSet) {
profileXML=toSet;
}
public GCatModel(InternalAlgorithmDescriptor desc) {
item=new CkanItem();
// item.setAuthor(desc.getAuthor());
// item.setAuthor(desc.getAuthor());
item.setTitle(desc.getName()+" in "+ContextUtils.getCurrentScopeName());
item.setLicense_id("CC-BY-NC-SA-4.0");
// item.setMaintainer(desc.getMaintainer());
// item.setMaintainer(desc.getMaintainer());
item.setName(item.getTitle().toLowerCase().toLowerCase().replaceAll(" ", "_"));
for(String tag:desc.getTags()) {
item.getTags().add(new CkanItem.Tag(fixTag(tag)));
@ -47,56 +50,62 @@ public class GCatModel implements CatalogueFormatData {
item.getTags().add(new CkanItem.Tag("WPS"));
item.getTags().add(new CkanItem.Tag("Analytics"));
item.getExtras().add(new CKanExtraField("system:type", profileID));
item.setPrivateFlag(desc.getPrivateFlag());
for(Parameter param: desc.getInputParameters())
item.getExtras().add(new CKanExtraField(profileID+":Input Parameter",
String.format("%1$s [%2$s] %3$s : %4$s",
param.getName(),param.getType(),
((param.getValue()!=null&&!param.getValue().isEmpty())?"default : "+param.getValue():""),
param.getDescription())));
for(Parameter param: desc.getOutputParameters())
item.getExtras().add(new CKanExtraField(profileID+":Output Parameter",
String.format("%1$s [%2$s] %3$s : %4$s",
param.getName(),param.getType(),
((param.getValue()!=null&&!param.getValue().isEmpty())?"default : "+param.getValue():""),
param.getDescription())));
//Algorithm Description
// item.getExtras().add(new CKanExtraField(profileID+":Process Description", desc.getDescription()));
// item.getExtras().add(new CKanExtraField(profileID+":Process Description", desc.getDescription()));
item.setNotes(desc.getDescription());
// Algorithm Users
item.getExtras().add(new CKanExtraField(profileID+":Process Author",desc.getAuthor().asStringValue()));
item.getExtras().add(new CKanExtraField(profileID+":Process Maintainer",desc.getAuthor().asStringValue()));
if(desc.getGuiLink()!=null)
resources.add(new CkanResource("Gateway Link",desc.getGuiLink(),"HTTP","Link to the GUI designed to operate with DataMiner"));
if(desc.getGuiLink()!=null) {
try {
URL url=new URL(desc.getGuiLink());
resources.add(new CkanResource("Gateway Link",desc.getGuiLink(),url.getProtocol(),"Link to the GUI designed to operate with DataMiner"));
}catch(Throwable t) {
log.warn("Unable to generate resource from gui Link : "+desc.getGuiLink(),t);
}
}
if(desc.getWpsLink()!=null)
resources.add(new CkanResource("WPS Link", desc.getWpsLink(), "WPS","WPS Link to the "+DataMinerCollectorProperties.getProperty(DataMinerCollectorProperties.CKAN_RESOURCE_TYPE)));
}
private String profile=profileXML;
private CkanItem item=null;
private ArrayList<CkanResource> resources=new ArrayList<>();
@Override
public String toCatalogueFormat() throws InternalConversionException {
try{
ByteArrayOutputStream baos=new ByteArrayOutputStream();
mapper.writeValue(baos, this);
return baos.toString();
mapper.writeValue(baos, this);
return baos.toString();
}catch(Throwable t) {
throw new InternalConversionException("Unable to convert",t);
}
@ -129,7 +138,7 @@ public class GCatModel implements CatalogueFormatData {
fixedTag=fixedTag.substring(0,96)+"...";
return fixedTag.trim();
}
static final String identityString(UserIdentity id) {
StringBuilder builder=new StringBuilder(id.getLastName()+", ");
builder.append(id.getFirstName()+", ");

10
gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/ExecutionTask.java
View File

@ -133,14 +133,14 @@ public class ExecutionTask implements Runnable {
log.info("Acquired : {} ",request);
ExecutionReport report=new ExecutionReport();
report.getGenericInformations().setStartTime(Instant.now());
report.setStartingScope(infrastructure.getCurrentContextName());
// -- ON SUCCESS reload request
request =persistence.getById(request.getId());
ExecutionReport report=new ExecutionReport();
report.getGenericInformations().setStartTime(request.getStartTime());
report.setStartingScope(infrastructure.getCurrentContextName());
// For ALL COLLECTORS IN REQUEST

2
gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/engine/impl/persistence/Queries.java
View File

@ -58,7 +58,7 @@ public class Queries {
fields.get(ID)});
public static final Query ACQUIRE= new Query("UPDATE "+TABLE+" SET "
+STATUS+"='"+ExecutionStatus.RUNNING+"' WHERE "+ID+"=? AND "+STATUS+"='"+ExecutionStatus.PENDING+"'",
+STATUS+"='"+ExecutionStatus.RUNNING+"', "+START+"=CURRENT_TIMESTAMP WHERE "+ID+"=? AND "+STATUS+"='"+ExecutionStatus.PENDING+"'",
new DBField[] {fields.get(ID)});
public static final Query GET_ALL= new Query("SELECT * FROM "+TABLE+" ORDER BY "+END+" DESC",

6
gCat-Feeder/src/main/java/org/gcube/data/publishing/gCatFeeder/service/model/ExecutionDescriptor.java
View File

@ -77,6 +77,12 @@ public class ExecutionDescriptor {
}
public void setStatus(ExecutionStatus status) {
this.status = status;
switch(status) {
case FAILED :
case STOPPED :
case SUCCESS : this.setEndTime(Instant.now());
break;
}
}
public String getReportUrl() {
return reportUrl;