gCubeSystem
/
software-versions-processor-lib
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implementing variable management

This commit is contained in:
Luca Frosini 2023-01-11 18:21:11 +01:00
parent ae391e637f
commit b3c5858fe9
9 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/org/gcube/common/software/analyser/SoftwareConceptAnalyser.java
View File

@ -36,7 +36,7 @@ public class SoftwareConceptAnalyser {
this.jsonNode = objectMapper.readTree(json);
}
protected JsonNode actualize(JsonNode version) throws Exception {
protected SoftwareVersion actualize(JsonNode version) throws Exception {
Variables variables = objectMapper.treeToValue(version, Variables.class);
Set<String> missingVariables = variables.parse();
int size = missingVariables.size();
@ -45,7 +45,9 @@ public class SoftwareConceptAnalyser {
missingVariables.toArray(new String[size]).toString());
}
JsonNode swVersion = objectMapper.convertValue(variables.getProperties(), JsonNode.class);
return swVersion;
SoftwareVersion softwareVersion = objectMapper.treeToValue(swVersion, SoftwareVersion.class);
softwareVersion.setVariables(variables);
return softwareVersion;
}
public void analyse() throws Exception {
@ -58,11 +60,12 @@ public class SoftwareConceptAnalyser {
for(int i=0; i<versions.size(); i++) {
JsonNode version = Utils.merge(concept, versions.get(i));
version = actualize(version);
SoftwareVersion softwareVersion = objectMapper.treeToValue(version, SoftwareVersion.class);
SoftwareVersion softwareVersion = actualize(version);
softwareVersion.setPrevious(previous);
previous.setNext(softwareVersion);
if(previous!=null) {
previous.setNext(softwareVersion);
}
logger.trace("Going to process {} {} (previous version {})",
name, softwareVersion.getVersion(),

17
src/main/java/org/gcube/common/software/analyser/SoftwareVersionAnalyser.java
View File

@ -1,13 +1,17 @@
package org.gcube.common.software.analyser;
import java.io.File;
import java.net.URL;
import java.util.List;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.software.model.SoftwareVersion;
import org.gcube.common.software.model.SoftwareVersionFile;
import org.gcube.common.software.model.Variables;
import org.gcube.common.software.process.export.SoftwareVersionExporter;
import org.gcube.common.software.process.publish.SoftwareVersionPublisher;
import org.gcube.common.software.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,10 +43,15 @@ public class SoftwareVersionAnalyser {
}
public void analyse() throws Exception {
List<SoftwareVersionFile> files = softwareVersion.getFiles();
for(SoftwareVersionFile file : files) {
Variables variables = softwareVersion.getVariables();
List<SoftwareVersionFile> svfs = softwareVersion.getFiles();
for(SoftwareVersionFile svf : svfs) {
URL url = svf.getURL();
String urlString = variables.replaceAllVariables(url.toString());
svf.setURL(new URL(urlString));
String desiredName = svf.getDesiredName();
desiredName = variables.replaceAllVariables(desiredName);
svf.setDesiredName(desiredName);
}
ObjectNode publishers = (ObjectNode) originalJson.get("publishers");

2
src/main/java/org/gcube/common/software/model/SoftwareConcept.java
View File

@ -58,7 +58,7 @@ public class SoftwareConcept {
@JsonProperty(DOI_URL_PROPERTY_NAME)
protected URL doiURL;
@JsonProperty(AUTHORS_PROPERTY_NAME)
@JsonProperty(GRANTS_PROPERTY_NAME)
protected JsonNode grants;
@JsonProperty(PUBLISH_PROPERTY_NAME)

26
src/main/java/org/gcube/common/software/model/SoftwareVersionFile.java
View File

@ -1,7 +1,14 @@
package org.gcube.common.software.model;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import org.gcube.com.fasterxml.jackson.annotation.JsonProperty;
import org.gcube.com.fasterxml.jackson.annotation.JsonSetter;
@ -39,4 +46,23 @@ public class SoftwareVersionFile {
this.desiredName = desiredName;
}
public File downloadFile() throws IOException {
File file = new File(desiredName);
Path path = Paths.get(desiredName);
try (InputStream inputStream = url.openStream()) {
Files.copy(inputStream, path, StandardCopyOption.REPLACE_EXISTING);
}
/* Uses apache common-io */
// FileUtils.copyURLToFile(df.getURL(), file);
if(!file.exists()) {
throw new RuntimeException(file.getAbsolutePath() + " does not exist");
}
if(file.length()==0) {
throw new RuntimeException(file.getAbsolutePath() + " has size 0");
}
return file;
}
}

18
src/main/java/org/gcube/common/software/model/Variables.java
View File

@ -49,6 +49,24 @@ public class Variables {
return variables;
}
public String replaceAllVariables(String value) throws Exception {
Set<String> missingVariables = new HashSet<>();
Set<String> variableNames = findVariables(value);
for(String variableName : variableNames) {
if(properties.containsKey(variableName)) {
String variableValue = properties.get(variableName).toString();
value = Utils.replaceVariable(variableName, variableValue, value);
}else {
missingVariables.add(variableName);
}
}
if(missingVariables.size()>0) {
throw new Exception("The following varibles cannot be replaced because they don't exists " + missingVariables.toString());
}
return value;
}
protected Set<String> replaceAllVariables(String key, String value) throws Exception {
Set<String> missingVariables = new HashSet<>();
Set<String> variableNames = findVariables(value);

20
src/main/java/org/gcube/common/software/process/export/biblatex/BibLaTeXSoftwareVersionExporter.java
View File

@ -10,6 +10,7 @@ import java.util.Set;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.common.software.model.ElaborationType;
import org.gcube.common.software.model.Variables;
import org.gcube.common.software.process.export.SoftwareVersionExporter;
import org.gcube.common.software.utils.FileUtils;
import org.gcube.common.software.utils.Utils;
@ -76,12 +77,6 @@ public class BibLaTeXSoftwareVersionExporter extends SoftwareVersionExporter {
return stringBuffer.toString();
}
private String getTitle(JsonNode metadata) {
String title = metadata.get("title").asText();
title = Utils.replaceVariable("version", softwareVersion.getVersion(), title);
return title;
}
private String getAuthors(ArrayNode arrayNode) {
StringBuffer stringBuffer = new StringBuffer();
int size = arrayNode.size();
@ -147,18 +142,15 @@ public class BibLaTeXSoftwareVersionExporter extends SoftwareVersionExporter {
// return s;
// }
protected String parseTemplate(String template) {
JsonNode metadata = null; // softwareConcept.getMetadata();
protected String parseTemplate(String template) throws Exception {
String s = Utils.replaceVariable("citation_id", getCitationID(), template);
s = Utils.replaceVariable("author", getAuthors(softwareVersion.getAuthors()), s);
s = Utils.replaceVariable("title", getTitle(metadata), s);
s = Utils.replaceVariable("description", softwareVersion.getDescription(), s);
s = Utils.replaceVariable("date", Utils.getDateAsString(softwareVersion.getDate()), s);
s = Utils.replaceVariable("version", softwareVersion.getVersion(), s);
s = Utils.replaceVariable("doi", softwareVersion.getVersionDOIURL().toString(), s);
s = Utils.replaceVariable("keywords", getKeywords(softwareVersion.getKeywords()), s);
// s = Utils.replaceVariable("license", getLicense(metadata), s);
Variables variables = softwareVersion.getVariables();
s = variables.replaceAllVariables(s);
// s = addNotes(s);
return s;
}

49
src/main/java/org/gcube/common/software/process/publish/zenodo/ZenodoSoftwareVersionPublisher.java
View File

@ -8,12 +8,9 @@ import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -95,26 +92,6 @@ public class ZenodoSoftwareVersionPublisher extends SoftwareVersionPublisher {
}
}
protected File downloadFile(SoftwareVersionFile df) throws IOException {
File file = new File(df.getDesiredName());
Path path = Paths.get(df.getDesiredName());
try (InputStream inputStream = df.getURL().openStream()) {
Files.copy(inputStream, path, StandardCopyOption.REPLACE_EXISTING);
}
/* Uses apache common-io */
// FileUtils.copyURLToFile(df.getURL(), file);
if(!file.exists()) {
throw new RuntimeException(file.getAbsolutePath() + " does not exist");
}
if(file.length()==0) {
throw new RuntimeException(file.getAbsolutePath() + " has size 0");
}
return file;
}
protected void addFilesToDeposition(List<File> files ) throws Exception {
String depositID = getZenodoIDFromDOIURL(softwareVersion.getVersionDOIURL());
String newFilePath = DEPOSTION_FILES_PATH.replace(":id", depositID);
@ -177,8 +154,8 @@ public class ZenodoSoftwareVersionPublisher extends SoftwareVersionPublisher {
protected void finalize() throws Exception {
List<File> files = new ArrayList<>();
for(SoftwareVersionFile df : softwareVersion.getFiles()) {
File file = downloadFile(df);
for(SoftwareVersionFile svf : softwareVersion.getFiles()) {
File file = svf.downloadFile();
files.add(file);
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
}
@ -305,11 +282,21 @@ public class ZenodoSoftwareVersionPublisher extends SoftwareVersionPublisher {
}
private ArrayNode getGrants(){
ArrayNode grants = softwareVersion.getGrants().deepCopy();
for(JsonNode g : grants) {
ObjectNode grant = (ObjectNode) g;
grant.remove("name");
grant.remove("url");
ObjectMapper objectMapper = Utils.getObjectMapper();
ArrayNode grants = objectMapper.createArrayNode();
ArrayNode arrayNode = (ArrayNode) configuration.get("skip_grants");
Set<String> idToSkip = new HashSet<>();
for(JsonNode idNode : arrayNode) {
idToSkip.add(idNode.asText());
}
for(JsonNode g : softwareVersion.getGrants()) {
String id = g.get("id").asText();
if(idToSkip.contains(id)) {
continue;
}
ObjectNode grant = objectMapper.createObjectNode();
grant.put("id", id);
grants.add(grant);
}
return grants;
}

2
src/main/resources/biblatex.template
View File

@ -4,6 +4,6 @@
abstract = {{{description}}},
date = {{{date}}},
version = {{{version}}},
url = {{{doi}}},
url = {{{version_doi_url}}},
keywords = {{{keywords}}}
}

5
src/test/resources/gcat-test-sandbox.json
View File

@ -128,7 +128,7 @@
"url": "https://cordis.europa.eu/project/id/871042"
}
],
"publish": "ALL",
"publish": "NONE",
"export": "ALL"
},
"versions": [
@ -322,7 +322,8 @@
"publishers": {
"ZenodoSoftwareVersionPublisher" :{
"html_description": "<p><a href=\"https://www.gcube-system.org/\">gCube</a> Catalogue (gCat) Service allows any client to publish items in the gCube Catalogue.</p>\n\n<p><a href=\"https://www.gcube-system.org/\">gCube</a> is an open-source software toolkit used for building and operating Hybrid Data Infrastructures enabling the dynamic deployment of Virtual Research Environments, such as the <a href=\"https://www.d4science.org/\">D4Science Infrastructure</a>, by favouring the realisation of reuse-oriented policies.</p>\n\n<p><a href=\"https://www.gcube-system.org/\">gCube</a> has been used to successfully build and operate infrastructures and virtual research environments for application domains ranging from biodiversity to environmental data management and cultural heritage.</p>\n\n<p><a href=\"https://www.gcube-system.org/\">gCube</a> offers components supporting typical data management workflows including data access, curation, processing, and visualisation on a large set of data typologies ranging from primary biodiversity data to geospatial and tabular data.</p>\n\n<p><a href=\"https://www.d4science.org/\">D4Science</a> is a Hybrid Data Infrastructure combining over 500 software components and integrating data from more than 50 different data providers into a coherent and managed system of hardware, software, and data resources. The D4Science infrastructure drastically reduces the cost of ownership, maintenance, and operation thanks to the exploitation of gCube.</p>\n\n<p>&nbsp;</p>",
"html_code_location": "\n\n<p>The official source code location of this software version is available at:</p>\n\n<p><a href=\"{{code_location}}\">{{code_location}}</a></p>"
"html_code_location": "\n\n<p>The official source code location of this software version is available at:</p>\n\n<p><a href=\"{{code_location}}\">{{code_location}}</a></p>",
"skip_grants": ["004260"]
}
},
"exporters": {