Refactoring library

This commit is contained in:
luca.frosini 2023-08-31 11:31:14 +02:00
parent 8897a25f4d
commit 85a36d12e5
12 changed files with 137 additions and 128 deletions

View File

@ -14,19 +14,29 @@ The library currently offers two processors:
Other processors can be easily added in the future by extending the ``SoftwareArtifactProcessor`` class. Other processors can be easily added in the future by extending the ``SoftwareArtifactProcessor`` class.
The Core of the Library
=======================
The core class of the library is ``Analyser``, which must be initialized with: The core class of the library is ``Analyser``, which must be initialized with:
* a configuration (JSON Object); * a configuration (JSON Object);
* a list of software artifacts described by their metadata (JSON Array). * a list of software artifacts described by their metadata (JSON Array).
The configuration must contain at least the list of exporters to be used and their configuration parameters. The configuration must contain:
* The list of processors to be used and their configuration parameters (required);
* A optional set of metadata which will be used as default metadata for all software artifacts defined in the list.
Exporter configuration requires at least the ``elaboration`` property, which can assume the following values Exporter configuration requires at least the ``elaboration`` property, which can assume the following values
(see ``ElaborationType`` enumerated): (see ``ElaborationType`` enumerated):
* **ALL**: The exporter analyses all the software artifacts; * **ALL**: The exporter analyses all the software artifacts;
* **UPDATE_ONLY**: The exporter analyes only the software artifact to be updated in the target; * **UPDATE_ONLY**: The exporter analyes only the software artifact to be updated in the target;
* **NEW**: The exporter analyses only the software artifact that does not yet exist in the target; * **NEW**: The exporter analyses only the software artifact that does not yet exist in the target;
* **NONE**: Does not export the software artifact in the target, but each software artifact is elaborated for the exporting without effectively doing it. It is a dry run. * **NONE**: Does not export the software artifact in the target, but each software artifact is elaborated without effectively exporting it. It is a dry run.
The processors are executed in the order they are defined. The processors are executed in the order they are defined.
A processor could produce metadata itself (e.g., the obtained Zenodo DOI). A processor could produce metadata itself (e.g., the obtained Zenodo DOI).
@ -34,7 +44,6 @@ The input metadata + the metadata generated by a processor are made available to
At the end of the processing phase, the library produces the actualized output of all the software artifact metadata. At the end of the processing phase, the library produces the actualized output of all the software artifact metadata.
Moreover, the configuration can optionally include an arbitrary set of metadata used as default metadata for all software artifacts defined in the list.
The list of software artifacts contains an arbitrary set of metadata. The list of software artifacts contains an arbitrary set of metadata.
It depends on the processor using certain metadata, among others. It depends on the processor using certain metadata, among others.

View File

@ -13,12 +13,12 @@ import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper; import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode; import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.software.export.SoftwareArtifactProcessor; import org.gcube.common.software.model.ProcessorConfig;
import org.gcube.common.software.model.ExporterConfig;
import org.gcube.common.software.model.GlobalConfig; import org.gcube.common.software.model.GlobalConfig;
import org.gcube.common.software.model.SoftwareArtifactConfig; import org.gcube.common.software.model.SoftwareArtifactMetadata;
import org.gcube.common.software.model.SoftwareArtifactFile; import org.gcube.common.software.model.SoftwareArtifactFile;
import org.gcube.common.software.model.Variables; import org.gcube.common.software.model.Variables;
import org.gcube.common.software.processor.SoftwareArtifactProcessor;
import org.gcube.common.software.utils.Utils; import org.gcube.common.software.utils.Utils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -34,7 +34,7 @@ public class Analyser {
protected File outputDirectory; protected File outputDirectory;
protected ObjectNode globalConfiguration; protected ObjectNode globalConfiguration;
protected ArrayNode versionConfigurations; protected ArrayNode artifactMetadataArray;
public Analyser() throws Exception { public Analyser() throws Exception {
this.objectMapper = Utils.getObjectMapper(); this.objectMapper = Utils.getObjectMapper();
@ -52,16 +52,16 @@ public class Analyser {
this.globalConfiguration = originalGlobal.deepCopy(); this.globalConfiguration = originalGlobal.deepCopy();
} }
public ArrayNode getVersionConfigurations() { public ArrayNode getArtifactMetadataArray() {
return versionConfigurations; return artifactMetadataArray;
} }
public void setVersionConfigurations(ArrayNode originalVersions) { public void setArtifactMetadataArray(ArrayNode original) {
this.versionConfigurations = originalVersions.deepCopy(); this.artifactMetadataArray = original.deepCopy();
} }
protected SoftwareArtifactConfig actualizeSoftwareVersionConfig(JsonNode version) throws Exception { protected SoftwareArtifactMetadata actualizeSoftwareArtifactConfig(JsonNode version) throws Exception {
((ObjectNode) version).remove(GlobalConfig.EXPORTERS_PROPERTY_NAME); ((ObjectNode) version).remove(GlobalConfig.PROCESSORS_PROPERTY_NAME);
Variables variables = objectMapper.treeToValue(version, Variables.class); Variables variables = objectMapper.treeToValue(version, Variables.class);
Set<String> missingVariables = variables.parse(); Set<String> missingVariables = variables.parse();
int size = missingVariables.size(); int size = missingVariables.size();
@ -70,7 +70,7 @@ public class Analyser {
missingVariables.toArray(new String[size]).toString()); missingVariables.toArray(new String[size]).toString());
} }
JsonNode swVersion = objectMapper.convertValue(variables.getProperties(), JsonNode.class); JsonNode swVersion = objectMapper.convertValue(variables.getProperties(), JsonNode.class);
SoftwareArtifactConfig softwareVersionConfig = objectMapper.treeToValue(swVersion, SoftwareArtifactConfig.class); SoftwareArtifactMetadata softwareVersionConfig = objectMapper.treeToValue(swVersion, SoftwareArtifactMetadata.class);
List<SoftwareArtifactFile> svfs = softwareVersionConfig.getFiles(); List<SoftwareArtifactFile> svfs = softwareVersionConfig.getFiles();
for(SoftwareArtifactFile svf : svfs) { for(SoftwareArtifactFile svf : svfs) {
@ -94,22 +94,22 @@ public class Analyser {
return globalConfig; return globalConfig;
} }
protected ExporterConfig actualizeExporterConfig(ExporterConfig exporterConfig, SoftwareArtifactConfig softwareVersionConfig) throws Exception { protected ProcessorConfig actualizeProcessorConfig(ProcessorConfig processorConfig, SoftwareArtifactMetadata softwareArtifactMetadata) throws Exception {
ObjectNode versionNode = objectMapper.valueToTree(softwareVersionConfig); ObjectNode versionNode = objectMapper.valueToTree(softwareArtifactMetadata);
Variables versionVariables = objectMapper.treeToValue(versionNode, Variables.class); Variables versionVariables = objectMapper.treeToValue(versionNode, Variables.class);
ObjectNode node = objectMapper.valueToTree(exporterConfig); ObjectNode node = objectMapper.valueToTree(processorConfig);
Variables variables = objectMapper.treeToValue(node, Variables.class); Variables variables = objectMapper.treeToValue(node, Variables.class);
variables.parseWith(versionVariables); variables.parseWith(versionVariables);
JsonNode ec = objectMapper.convertValue(variables.getProperties(), JsonNode.class); JsonNode ec = objectMapper.convertValue(variables.getProperties(), JsonNode.class);
return objectMapper.treeToValue(ec, ExporterConfig.class); return objectMapper.treeToValue(ec, ProcessorConfig.class);
} }
protected void checkExporters(Set<String> availableExporterNames, Set<String> requestedExporterNames) throws Exception { protected void checkProcessors(Set<String> availableProcessorNames, Set<String> requestedProcessorNames) throws Exception {
if(!availableExporterNames.containsAll(requestedExporterNames)) { if(!availableProcessorNames.containsAll(requestedProcessorNames)) {
requestedExporterNames.removeAll(availableExporterNames); requestedProcessorNames.removeAll(availableProcessorNames);
throw new Exception("The following requested exporters does not exists " + requestedExporterNames); throw new Exception("The following requested exporters does not exists " + requestedProcessorNames);
} }
} }
@ -118,9 +118,9 @@ public class Analyser {
GlobalConfig globalConfig = getGlobalConfig(globalConfiguration); GlobalConfig globalConfig = getGlobalConfig(globalConfiguration);
Map<String, Class<? extends SoftwareArtifactProcessor>> availableExporters = SoftwareArtifactProcessor.getAvailableExporters(); Map<String, Class<? extends SoftwareArtifactProcessor>> availableProcessors = SoftwareArtifactProcessor.getAvailableProcessors();
Map<String,ExporterConfig> requestedExporters = globalConfig.getExporters(); Map<String,ProcessorConfig> requestedProcessors = globalConfig.getProcessorConfigurations();
checkExporters(availableExporters.keySet(), requestedExporters.keySet()); checkProcessors(availableProcessors.keySet(), requestedProcessors.keySet());
if(outputDirectory==null) { if(outputDirectory==null) {
outputDirectory = new File(globalConfig.getFileName()); outputDirectory = new File(globalConfig.getFileName());
@ -130,37 +130,37 @@ public class Analyser {
Files.createDirectories(outputDirectory.toPath()); Files.createDirectories(outputDirectory.toPath());
} }
SoftwareArtifactConfig previous = null; SoftwareArtifactMetadata previous = null;
int i = 0; int i = 0;
List<File> outputFiles = new ArrayList<>(); List<File> outputFiles = new ArrayList<>();
for(i=0; i<versionConfigurations.size(); i++) { for(i=0; i<artifactMetadataArray.size(); i++) {
ObjectNode versionConfig = (ObjectNode) versionConfigurations.get(i).deepCopy(); ObjectNode artifactMetadata = (ObjectNode) artifactMetadataArray.get(i).deepCopy();
JsonNode mergedVersionConfig = Utils.merge(globalConfiguration, versionConfig); JsonNode mergedArtifactMetadata = Utils.merge(globalConfiguration, artifactMetadata);
SoftwareArtifactConfig softwareVersionConfig = actualizeSoftwareVersionConfig(mergedVersionConfig); SoftwareArtifactMetadata softwareArtifactMetadata = actualizeSoftwareArtifactConfig(mergedArtifactMetadata);
softwareVersionConfig.setOriginalJson(versionConfig); softwareArtifactMetadata.setOriginalJson(artifactMetadata);
softwareVersionConfig.setPrevious(previous); softwareArtifactMetadata.setPrevious(previous);
logger.trace("Going to process {}", softwareVersionConfig.getTitle()); logger.trace("Going to process {}", softwareArtifactMetadata.getTitle());
for(String className : requestedExporters.keySet()) { for(String className : requestedProcessors.keySet()) {
logger.debug("Going to export with {}", className); logger.debug("Going to export with {}", className);
Class<? extends SoftwareArtifactProcessor> exporterClass = availableExporters.get(className); Class<? extends SoftwareArtifactProcessor> processorClass = availableProcessors.get(className);
ExporterConfig exporterConfig = requestedExporters.get(className); ProcessorConfig processorConfig = requestedProcessors.get(className);
exporterConfig = actualizeExporterConfig(exporterConfig, softwareVersionConfig); processorConfig = actualizeProcessorConfig(processorConfig, softwareArtifactMetadata);
SoftwareArtifactProcessor sve = exporterClass.newInstance(); SoftwareArtifactProcessor sve = processorClass.newInstance();
sve.setOutputDirectory(outputDirectory); sve.setOutputDirectory(outputDirectory);
sve.setGlobalConfig(globalConfig); sve.setGlobalConfig(globalConfig);
sve.setSoftwareVersionConfig(softwareVersionConfig); sve.setSoftwareArtifactConfig(softwareArtifactMetadata);
sve.setExporterConfig(exporterConfig); sve.setExporterConfig(processorConfig);
sve.setFirst(i==0); sve.setFirst(i==0);
boolean last = i==(versionConfigurations.size()-1); boolean last = i==(artifactMetadataArray.size()-1);
sve.setLast(last); sve.setLast(last);
sve.export(); sve.export();
@ -171,7 +171,7 @@ public class Analyser {
Thread.sleep(TimeUnit.SECONDS.toMillis(2)); Thread.sleep(TimeUnit.SECONDS.toMillis(2));
previous = softwareVersionConfig; previous = softwareArtifactMetadata;
} }
return outputFiles; return outputFiles;

View File

@ -32,10 +32,10 @@ public class AnalyserFactory {
public static Analyser getAnalyser(JsonNode jsonNode) throws Exception { public static Analyser getAnalyser(JsonNode jsonNode) throws Exception {
Analyser analyser = new Analyser(); Analyser analyser = new Analyser();
ObjectNode originalGlobal = (ObjectNode) jsonNode.get(CONFIGURATION_PROPERTY_NAME); ObjectNode originalGlobalConfiguration = (ObjectNode) jsonNode.get(CONFIGURATION_PROPERTY_NAME);
analyser.setGlobalConfiguration(originalGlobal); analyser.setGlobalConfiguration(originalGlobalConfiguration);
ArrayNode originalVersions = (ArrayNode) jsonNode.get(ARTIFACTS_PROPERTY_NAME); ArrayNode originalArtifactMetadataArray = (ArrayNode) jsonNode.get(ARTIFACTS_PROPERTY_NAME);
analyser.setVersionConfigurations(originalVersions); analyser.setArtifactMetadataArray(originalArtifactMetadataArray);
return analyser; return analyser;
} }

View File

@ -20,10 +20,10 @@ public class GlobalConfig {
public static final String EXPORT_FILENAME_PROPERTY_NAME = "export_filename"; public static final String EXPORT_FILENAME_PROPERTY_NAME = "export_filename";
public static final String EXPORTERS_PROPERTY_NAME = "exporters"; public static final String PROCESSORS_PROPERTY_NAME = "processors";
@JsonProperty(EXPORTERS_PROPERTY_NAME) @JsonProperty(PROCESSORS_PROPERTY_NAME)
protected Map<String,ExporterConfig> exporters; protected Map<String,ProcessorConfig> processorConfigurations;
protected Map<String, JsonNode> properties; protected Map<String, JsonNode> properties;
@ -34,8 +34,8 @@ public class GlobalConfig {
this.properties = new LinkedHashMap<>(); this.properties = new LinkedHashMap<>();
} }
public Map<String,ExporterConfig> getExporters() { public Map<String,ProcessorConfig> getProcessorConfigurations() {
return exporters; return processorConfigurations;
} }
@JsonIgnore @JsonIgnore

View File

@ -13,7 +13,7 @@ import org.gcube.com.fasterxml.jackson.databind.JsonNode;
/** /**
* @author Luca Frosini (ISTI - CNR) * @author Luca Frosini (ISTI - CNR)
*/ */
public class ExporterConfig { public class ProcessorConfig {
public static final String ELABORATION_PROPERTY_NAME = "elaboration"; public static final String ELABORATION_PROPERTY_NAME = "elaboration";
@ -23,11 +23,11 @@ public class ExporterConfig {
protected Map<String, JsonNode> properties; protected Map<String, JsonNode> properties;
public ExporterConfig() { public ProcessorConfig() {
properties = new LinkedHashMap<>(); properties = new LinkedHashMap<>();
} }
public ElaborationType getElaboration() { public ElaborationType getElaborationType() {
return elaboration; return elaboration;
} }

View File

@ -21,7 +21,7 @@ import org.gcube.common.software.utils.Utils;
* @author Luca Frosini (ISTI - CNR) * @author Luca Frosini (ISTI - CNR)
* TO BE GENERALIZED * TO BE GENERALIZED
*/ */
public class SoftwareArtifactConfig { public class SoftwareArtifactMetadata {
public static final String PREVIOUS_CONCEPT_DOI_VALUE = "PREVIOUS"; public static final String PREVIOUS_CONCEPT_DOI_VALUE = "PREVIOUS";
@ -40,10 +40,10 @@ public class SoftwareArtifactConfig {
public static final String GRANTS_PROPERTY_NAME = "grants"; public static final String GRANTS_PROPERTY_NAME = "grants";
@JsonIgnore @JsonIgnore
protected SoftwareArtifactConfig previous; protected SoftwareArtifactMetadata previous;
@JsonIgnore @JsonIgnore
protected SoftwareArtifactConfig next; protected SoftwareArtifactMetadata next;
@JsonIgnore @JsonIgnore
protected Boolean newDeposition; protected Boolean newDeposition;
@ -92,18 +92,18 @@ public class SoftwareArtifactConfig {
protected Map<String, JsonNode> additionalProperties; protected Map<String, JsonNode> additionalProperties;
public SoftwareArtifactConfig() { public SoftwareArtifactMetadata() {
this.newDeposition = false; this.newDeposition = false;
this.additionalProperties = new LinkedHashMap<>(); this.additionalProperties = new LinkedHashMap<>();
} }
@JsonIgnore @JsonIgnore
public SoftwareArtifactConfig getPrevious() { public SoftwareArtifactMetadata getPrevious() {
return previous; return previous;
} }
@JsonIgnore @JsonIgnore
public void setPrevious(SoftwareArtifactConfig previous) { public void setPrevious(SoftwareArtifactMetadata previous) {
this.previous = previous; this.previous = previous;
if(previous!=null) { if(previous!=null) {
this.previous.next = this; this.previous.next = this;
@ -111,7 +111,7 @@ public class SoftwareArtifactConfig {
} }
@JsonIgnore @JsonIgnore
public SoftwareArtifactConfig getNext() { public SoftwareArtifactMetadata getNext() {
return next; return next;
} }

View File

@ -1,14 +1,14 @@
package org.gcube.common.software.export; package org.gcube.common.software.processor;
import java.io.File; import java.io.File;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.gcube.common.software.export.biblatex.BibLaTeXExporter; import org.gcube.common.software.model.ProcessorConfig;
import org.gcube.common.software.export.zenodo.ZenodoExporter;
import org.gcube.common.software.model.ExporterConfig;
import org.gcube.common.software.model.GlobalConfig; import org.gcube.common.software.model.GlobalConfig;
import org.gcube.common.software.model.SoftwareArtifactConfig; import org.gcube.common.software.model.SoftwareArtifactMetadata;
import org.gcube.common.software.processor.biblatex.BibLaTeXExporter;
import org.gcube.common.software.processor.zenodo.ZenodoExporter;
/** /**
* @author Luca Frosini (ISTI - CNR) * @author Luca Frosini (ISTI - CNR)
@ -27,14 +27,14 @@ public abstract class SoftwareArtifactProcessor {
availableProcessors.put(clz.getSimpleName(), clz); availableProcessors.put(clz.getSimpleName(), clz);
} }
public static Map<String, Class<? extends SoftwareArtifactProcessor>> getAvailableExporters() { public static Map<String, Class<? extends SoftwareArtifactProcessor>> getAvailableProcessors() {
return availableProcessors; return availableProcessors;
} }
protected File outputDirectory; protected File outputDirectory;
protected GlobalConfig globalConfig; protected GlobalConfig globalConfig;
protected SoftwareArtifactConfig softwareVersionConfig; protected SoftwareArtifactMetadata softwareArtifactMetadata;
protected ExporterConfig exporterConfig; protected ProcessorConfig processorConfig;
protected boolean first; protected boolean first;
protected boolean last; protected boolean last;
@ -57,20 +57,20 @@ public abstract class SoftwareArtifactProcessor {
this.globalConfig = globalConfig; this.globalConfig = globalConfig;
} }
public SoftwareArtifactConfig getSoftwareVersionConfig() { public SoftwareArtifactMetadata getSoftwareArtifactConfig() {
return softwareVersionConfig; return softwareArtifactMetadata;
} }
public void setSoftwareVersionConfig(SoftwareArtifactConfig softwareVersionConfig) { public void setSoftwareArtifactConfig(SoftwareArtifactMetadata softwareArtifactMetadata) {
this.softwareVersionConfig = softwareVersionConfig; this.softwareArtifactMetadata = softwareArtifactMetadata;
} }
public ExporterConfig getExporterConfig() { public ProcessorConfig getExporterConfig() {
return exporterConfig; return processorConfig;
} }
public void setExporterConfig(ExporterConfig processorConfig) { public void setExporterConfig(ProcessorConfig processorConfig) {
this.exporterConfig = processorConfig; this.processorConfig = processorConfig;
} }
public void setFirst(boolean first) { public void setFirst(boolean first) {

View File

@ -1,4 +1,4 @@
package org.gcube.common.software.export.biblatex; package org.gcube.common.software.processor.biblatex;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.File; import java.io.File;
@ -9,9 +9,9 @@ import java.util.Set;
import org.gcube.com.fasterxml.jackson.databind.JsonNode; import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode; import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.common.software.export.SoftwareArtifactProcessor;
import org.gcube.common.software.model.ElaborationType; import org.gcube.common.software.model.ElaborationType;
import org.gcube.common.software.model.Variables; import org.gcube.common.software.model.Variables;
import org.gcube.common.software.processor.SoftwareArtifactProcessor;
import org.gcube.common.software.utils.FileUtils; import org.gcube.common.software.utils.FileUtils;
import org.gcube.common.software.utils.Utils; import org.gcube.common.software.utils.Utils;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -41,9 +41,9 @@ public class BibLaTeXExporter extends SoftwareArtifactProcessor {
exportFile.createNewFile(); exportFile.createNewFile();
} }
String title = softwareVersionConfig.getTitle(); String title = softwareArtifactMetadata.getTitle();
ElaborationType export = exporterConfig.getElaboration(); ElaborationType export = processorConfig.getElaborationType();
switch (export) { switch (export) {
case ALL: case ALL:
@ -51,7 +51,7 @@ public class BibLaTeXExporter extends SoftwareArtifactProcessor {
break; break;
case UPDATE_ONLY: case UPDATE_ONLY:
if (softwareVersionConfig.isNewDeposition()) { if (softwareArtifactMetadata.isNewDeposition()) {
logger.info("Skipping export for {}.", title); logger.info("Skipping export for {}.", title);
return; return;
} }
@ -59,7 +59,7 @@ public class BibLaTeXExporter extends SoftwareArtifactProcessor {
break; break;
case NEW: case NEW:
if (!softwareVersionConfig.isNewDeposition()) { if (!softwareArtifactMetadata.isNewDeposition()) {
logger.info("Skipping export for {}.", title); logger.info("Skipping export for {}.", title);
return; return;
} }
@ -157,10 +157,10 @@ public class BibLaTeXExporter extends SoftwareArtifactProcessor {
protected String parseTemplate(String template) throws Exception { protected String parseTemplate(String template) throws Exception {
String s = template; String s = template;
s = Utils.replaceVariable("author", getAuthors(softwareVersionConfig.getAuthors()), s); s = Utils.replaceVariable("author", getAuthors(softwareArtifactMetadata.getAuthors()), s);
s = Utils.replaceVariable("keywords", getKeywords(softwareVersionConfig.getKeywords()), s); s = Utils.replaceVariable("keywords", getKeywords(softwareArtifactMetadata.getKeywords()), s);
Variables variables = softwareVersionConfig.getVariables(); Variables variables = softwareArtifactMetadata.getVariables();
s = variables.replaceAllVariables(s); s = variables.replaceAllVariables(s);
// s = addNotes(s); // s = addNotes(s);
@ -168,8 +168,8 @@ public class BibLaTeXExporter extends SoftwareArtifactProcessor {
} }
protected void generate() throws Exception { protected void generate() throws Exception {
String title = softwareVersionConfig.getTitle(); String title = softwareArtifactMetadata.getTitle();
if(softwareVersionConfig.getVersionDOIURL()==null) { if(softwareArtifactMetadata.getVersionDOIURL()==null) {
logger.info("No Version DOI URL for {}. It will not be exported in BibLaTex format.", title); logger.info("No Version DOI URL for {}. It will not be exported in BibLaTex format.", title);
return; return;
} }

View File

@ -1,4 +1,4 @@
package org.gcube.common.software.export.zenodo; package org.gcube.common.software.processor.zenodo;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
@ -30,9 +30,9 @@ import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest; import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.gcube.common.software.analyser.AnalyserFactory; import org.gcube.common.software.analyser.AnalyserFactory;
import org.gcube.common.software.config.Config; import org.gcube.common.software.config.Config;
import org.gcube.common.software.export.SoftwareArtifactProcessor;
import org.gcube.common.software.model.ElaborationType; import org.gcube.common.software.model.ElaborationType;
import org.gcube.common.software.model.SoftwareArtifactConfig; import org.gcube.common.software.model.SoftwareArtifactMetadata;
import org.gcube.common.software.processor.SoftwareArtifactProcessor;
import org.gcube.common.software.model.SoftwareArtifactFile; import org.gcube.common.software.model.SoftwareArtifactFile;
import org.gcube.common.software.utils.Utils; import org.gcube.common.software.utils.Utils;
import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.ClientProperties;
@ -97,7 +97,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
} }
protected void addFilesToDeposition(List<File> files ) throws Exception { protected void addFilesToDeposition(List<File> files ) throws Exception {
String depositID = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL()); String depositID = getZenodoIDFromDOIURL(softwareArtifactMetadata.getVersionDOIURL());
String newFilePath = DEPOSTION_FILES_PATH.replace(":id", depositID); String newFilePath = DEPOSTION_FILES_PATH.replace(":id", depositID);
URL url = new URL(zenodoBaseURL, newFilePath); URL url = new URL(zenodoBaseURL, newFilePath);
@ -132,7 +132,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
gxHTTPStringRequest.header("Content-Type", "application/json"); gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json"); gxHTTPStringRequest.header("Accept", "application/json");
String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL()); String id = getZenodoIDFromDOIURL(softwareArtifactMetadata.getVersionDOIURL());
gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", id)); gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", id));
ObjectNode metadata = generateMetadata(); ObjectNode metadata = generateMetadata();
@ -149,7 +149,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
gxHTTPStringRequest.header("Content-Type", "application/json"); gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json"); gxHTTPStringRequest.header("Accept", "application/json");
String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL()); String id = getZenodoIDFromDOIURL(softwareArtifactMetadata.getVersionDOIURL());
gxHTTPStringRequest.path(DEPOSTION_PUBLISH_PATH.replace(":id", id)); gxHTTPStringRequest.path(DEPOSTION_PUBLISH_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post(); HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
@ -158,7 +158,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
protected void finalize() throws Exception { protected void finalize() throws Exception {
List<File> files = new ArrayList<>(); List<File> files = new ArrayList<>();
for(SoftwareArtifactFile svf : softwareVersionConfig.getFiles()) { for(SoftwareArtifactFile svf : softwareArtifactMetadata.getFiles()) {
File file = svf.downloadFile(); File file = svf.downloadFile();
files.add(file); files.add(file);
Thread.sleep(TimeUnit.SECONDS.toMillis(1)); Thread.sleep(TimeUnit.SECONDS.toMillis(1));
@ -273,23 +273,23 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
response = getResponse(httpURLConnection); response = getResponse(httpURLConnection);
String conceptDOIURL = createZenodoDOIURLFromID(response.get("conceptrecid").asText()); String conceptDOIURL = createZenodoDOIURLFromID(response.get("conceptrecid").asText());
softwareVersionConfig.setConceptDOIURL(conceptDOIURL); softwareArtifactMetadata.setConceptDOIURL(conceptDOIURL);
String versionDOIURL = createZenodoDOIURLFromID(response.get("id").asText()); String versionDOIURL = createZenodoDOIURLFromID(response.get("id").asText());
softwareVersionConfig.setVersionDOIURL(versionDOIURL); softwareArtifactMetadata.setVersionDOIURL(versionDOIURL);
finalize(); finalize();
} }
private ArrayNode getAuthors(){ private ArrayNode getAuthors(){
ArrayNode authors = softwareVersionConfig.getAuthors().deepCopy(); ArrayNode authors = softwareArtifactMetadata.getAuthors().deepCopy();
return authors; return authors;
} }
private String getDescription() { private String getDescription() {
StringBuffer stringBuffer = new StringBuffer(); StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append(softwareVersionConfig.getAdditionalProperty(HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText()); stringBuffer.append(softwareArtifactMetadata.getAdditionalProperty(HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText());
if(exporterConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME)!=null) { if(processorConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME)!=null) {
String additionalHTMLDescription = exporterConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText(); String additionalHTMLDescription = processorConfig.getProperty(ADDITIONAL_HTML_DESCRIPTION_CONFIG_FIELD_NAME).asText();
stringBuffer.append(additionalHTMLDescription); stringBuffer.append(additionalHTMLDescription);
} }
@ -299,12 +299,12 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
private ArrayNode getGrants(){ private ArrayNode getGrants(){
ObjectMapper objectMapper = Utils.getObjectMapper(); ObjectMapper objectMapper = Utils.getObjectMapper();
ArrayNode grants = objectMapper.createArrayNode(); ArrayNode grants = objectMapper.createArrayNode();
ArrayNode arrayNode = (ArrayNode) exporterConfig.getProperty(SKIP_GRANTS_CONFIG_FIELD_NAME); ArrayNode arrayNode = (ArrayNode) processorConfig.getProperty(SKIP_GRANTS_CONFIG_FIELD_NAME);
Set<String> idToSkip = new HashSet<>(); Set<String> idToSkip = new HashSet<>();
for(JsonNode idNode : arrayNode) { for(JsonNode idNode : arrayNode) {
idToSkip.add(idNode.asText()); idToSkip.add(idNode.asText());
} }
for(JsonNode g : softwareVersionConfig.getGrants()) { for(JsonNode g : softwareArtifactMetadata.getGrants()) {
String id = g.get("id").asText(); String id = g.get("id").asText();
if(idToSkip.contains(id)) { if(idToSkip.contains(id)) {
continue; continue;
@ -317,7 +317,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
} }
private ArrayNode getKeywords(){ private ArrayNode getKeywords(){
Set<String> keywords = softwareVersionConfig.getKeywords(); Set<String> keywords = softwareArtifactMetadata.getKeywords();
ObjectMapper objectMapper = Utils.getObjectMapper(); ObjectMapper objectMapper = Utils.getObjectMapper();
ArrayNode keywordsArrayNode = objectMapper.createArrayNode(); ArrayNode keywordsArrayNode = objectMapper.createArrayNode();
for(String keyword : keywords) { for(String keyword : keywords) {
@ -327,15 +327,15 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
} }
private ArrayNode getCommunities() { private ArrayNode getCommunities() {
return (ArrayNode) softwareVersionConfig.getAdditionalProperty(COMMUNITIES_FIELD_NAME); return (ArrayNode) softwareArtifactMetadata.getAdditionalProperty(COMMUNITIES_FIELD_NAME);
} }
private String getLicense() { private String getLicense() {
return softwareVersionConfig.getLicense().get("id").asText(); return softwareArtifactMetadata.getLicense().get("id").asText();
} }
private String getDate() { private String getDate() {
return Utils.getDateAsString(softwareVersionConfig.getDate()); return Utils.getDateAsString(softwareArtifactMetadata.getDate());
} }
private ObjectNode generateMetadata() { private ObjectNode generateMetadata() {
@ -352,8 +352,8 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
metadata.replace("keywords", getKeywords()); metadata.replace("keywords", getKeywords());
metadata.put("license", getLicense()); metadata.put("license", getLicense());
metadata.put("publication_date", getDate()); metadata.put("publication_date", getDate());
metadata.put("title", softwareVersionConfig.getTitle()); metadata.put("title", softwareArtifactMetadata.getTitle());
metadata.put("version", softwareVersionConfig.getVersion()); metadata.put("version", softwareArtifactMetadata.getVersion());
metadatWrapper.set(METADATA_FIELD_NAME, metadata); metadatWrapper.set(METADATA_FIELD_NAME, metadata);
return metadatWrapper; return metadatWrapper;
@ -366,7 +366,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT); gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters()); gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Accept", "application/json"); gxHTTPStringRequest.header("Accept", "application/json");
String id = getZenodoIDFromDOIURL(softwareVersionConfig.getVersionDOIURL()); String id = getZenodoIDFromDOIURL(softwareArtifactMetadata.getVersionDOIURL());
gxHTTPStringRequest.path(DEPOSTION_EDIT_PATH.replace(":id", id)); gxHTTPStringRequest.path(DEPOSTION_EDIT_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post(); HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
getResponse(httpURLConnection); getResponse(httpURLConnection);
@ -406,7 +406,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
gxHTTPStringRequest.header("Content-Type", "application/json"); gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json"); gxHTTPStringRequest.header("Accept", "application/json");
String conceptDOIURL = softwareVersionConfig.getConceptDOIURL(); String conceptDOIURL = softwareArtifactMetadata.getConceptDOIURL();
String conceptID = getZenodoIDFromDOIURL(conceptDOIURL); String conceptID = getZenodoIDFromDOIURL(conceptDOIURL);
gxHTTPStringRequest.path(RECORD_PATH.replace(":id", conceptID)); gxHTTPStringRequest.path(RECORD_PATH.replace(":id", conceptID));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.get(); HttpURLConnection httpURLConnection = gxHTTPStringRequest.get();
@ -419,13 +419,13 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
* this should avoid errors on softwareConcept. * this should avoid errors on softwareConcept.
*/ */
String latestVersionDOI = jsonNode.get("links").get("doi").asText(); String latestVersionDOI = jsonNode.get("links").get("doi").asText();
String previousVersionDOI = softwareVersionConfig.getPrevious().getVersionDOIURL().toString(); String previousVersionDOI = softwareArtifactMetadata.getPrevious().getVersionDOIURL().toString();
if(previousVersionDOI.compareTo(latestVersionDOI)!=0) { if(previousVersionDOI.compareTo(latestVersionDOI)!=0) {
logger.error("Zenodo obtained latest DOI {} != {} DOI from previous version", latestVersionDOI, previousVersionDOI); logger.error("Zenodo obtained latest DOI {} != {} DOI from previous version", latestVersionDOI, previousVersionDOI);
throw new RuntimeException("It seems that your json is not up to date with Zenodo."); throw new RuntimeException("It seems that your json is not up to date with Zenodo.");
} }
String latestVersionVersion = jsonNode.get("metadata").get("version").asText(); String latestVersionVersion = jsonNode.get("metadata").get("version").asText();
String previousVersionVersion = softwareVersionConfig.getPrevious().getVersion().toString(); String previousVersionVersion = softwareArtifactMetadata.getPrevious().getVersion().toString();
if(latestVersionVersion.compareTo(previousVersionVersion)!=0) { if(latestVersionVersion.compareTo(previousVersionVersion)!=0) {
logger.error("Zenodo obtained latest Version {} != {} Version from previous version", latestVersionVersion, previousVersionVersion); logger.error("Zenodo obtained latest Version {} != {} Version from previous version", latestVersionVersion, previousVersionVersion);
throw new RuntimeException("It seems that your json is not up to date with Zenodo."); throw new RuntimeException("It seems that your json is not up to date with Zenodo.");
@ -461,7 +461,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
// The reserved DOI of this created new version will be // The reserved DOI of this created new version will be
String newVersionDOIURL = response.get("doi_url").asText(); String newVersionDOIURL = response.get("doi_url").asText();
softwareVersionConfig.setVersionDOIURL(newVersionDOIURL); softwareArtifactMetadata.setVersionDOIURL(newVersionDOIURL);
// Remove previous depositionFiles // Remove previous depositionFiles
deletePreviousFiles(); deletePreviousFiles();
@ -471,7 +471,7 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
protected String getConfig(String propertyName) throws Exception { protected String getConfig(String propertyName) throws Exception {
String conf = null; String conf = null;
JsonNode node = exporterConfig.getProperty(propertyName); JsonNode node = processorConfig.getProperty(propertyName);
if(node == null || node.getNodeType()==JsonNodeType.NULL) { if(node == null || node.getNodeType()==JsonNodeType.NULL) {
conf = Config.getProperties().getProperty(propertyName); conf = Config.getProperties().getProperty(propertyName);
} }
@ -499,18 +499,18 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
getZenodoConnectionConfig(); getZenodoConnectionConfig();
String title = softwareVersionConfig.getTitle(); String title = softwareArtifactMetadata.getTitle();
ElaborationType publish = exporterConfig.getElaboration(); ElaborationType publish = processorConfig.getElaborationType();
if(publish==ElaborationType.NONE) { if(publish==ElaborationType.NONE) {
logger.info("Zenodo Deposit is disabled for {}.",title); logger.info("Zenodo Deposit is disabled for {}.",title);
return; return;
} }
if(softwareVersionConfig.getVersionDOIURL()!=null) { if(softwareArtifactMetadata.getVersionDOIURL()!=null) {
softwareVersionConfig.setNewDeposition(false); softwareArtifactMetadata.setNewDeposition(false);
if(publish==ElaborationType.ALL || if(publish==ElaborationType.ALL ||
publish==ElaborationType.UPDATE_ONLY) { publish==ElaborationType.UPDATE_ONLY) {
@ -525,9 +525,9 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
publish==ElaborationType.NEW) { publish==ElaborationType.NEW) {
logger.info("Going to deposit {}", title); logger.info("Going to deposit {}", title);
softwareVersionConfig.setNewDeposition(true); softwareArtifactMetadata.setNewDeposition(true);
if(softwareVersionConfig.getConceptDOIURL()==null) { if(softwareArtifactMetadata.getConceptDOIURL()==null) {
create(); create();
}else { }else {
newVersion(); newVersion();
@ -542,16 +542,16 @@ public class ZenodoExporter extends SoftwareArtifactProcessor {
ObjectNode toBeExported = objectMapper.createObjectNode(); ObjectNode toBeExported = objectMapper.createObjectNode();
toBeExported.replace(AnalyserFactory.CONFIGURATION_PROPERTY_NAME, globalConfig.getOriginalJson().deepCopy()); toBeExported.replace(AnalyserFactory.CONFIGURATION_PROPERTY_NAME, globalConfig.getOriginalJson().deepCopy());
ArrayNode array = objectMapper.createArrayNode(); ArrayNode array = objectMapper.createArrayNode();
SoftwareArtifactConfig previous = softwareVersionConfig; SoftwareArtifactMetadata previous = softwareArtifactMetadata;
boolean firstNode = true; boolean firstNode = true;
while(previous!=null){ while(previous!=null){
ObjectNode node = previous.getOriginalJson().deepCopy(); ObjectNode node = previous.getOriginalJson().deepCopy();
node.put(SoftwareArtifactConfig.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL()); node.put(SoftwareArtifactMetadata.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL());
if(firstNode) { if(firstNode) {
toBeExported.put(SoftwareArtifactConfig.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL()); toBeExported.put(SoftwareArtifactMetadata.CONCEPT_DOI_URL_PROPERTY_NAME, previous.getConceptDOIURL());
firstNode = false; firstNode = false;
} }
node.put(SoftwareArtifactConfig.VERSION_DOI_URL_PROPERTY_NAME, previous.getVersionDOIURL()); node.put(SoftwareArtifactMetadata.VERSION_DOI_URL_PROPERTY_NAME, previous.getVersionDOIURL());
array.insert(0, node); array.insert(0, node);
previous = previous.getPrevious(); previous = previous.getPrevious();
} }

View File

@ -3,7 +3,7 @@ package org.gcube.common.software.utils;
import java.io.File; import java.io.File;
import java.net.URL; import java.net.URL;
import org.gcube.common.software.export.biblatex.BibLaTeXExporter; import org.gcube.common.software.processor.biblatex.BibLaTeXExporter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View File

@ -135,7 +135,7 @@
} }
], ],
"export_filename": "{{name}}", "export_filename": "{{name}}",
"exporters": { "processors": {
"ZenodoExporter": { "ZenodoExporter": {
"elaboration": "NONE", "elaboration": "NONE",
"skip_grants": ["004260"], "skip_grants": ["004260"],

View File

@ -136,7 +136,7 @@
} }
], ],
"export_filename": "{{name}}", "export_filename": "{{name}}",
"exporters": { "processors": {
"ZenodoExporter": { "ZenodoExporter": {
"elaboration": "NONE", "elaboration": "NONE",
"skip_grants": ["004260"], "skip_grants": ["004260"],