gCubeSystem
/
software-versions-processor-lib
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implementing the solution

This commit is contained in:
Luca Frosini 2023-01-04 14:25:56 +01:00
parent 1c7b7427e1
commit 29431f67ac
7 changed files with 275 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -55,6 +55,18 @@
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.6</version>
</dependency>
<!-- Test libraries -->
<dependency>
<groupId>junit</groupId>

14
src/main/java/org/gcube/common/deposition/ElaborateDepositionVersion.java
View File

@ -178,22 +178,18 @@ public class ElaborateDepositionVersion {
logger.trace("DepositionVersion to be managed is:{}", objectMapper.writeValueAsString(depositionVersion));
DepositionVersionExecutor dve = DepositionVersionExecutor.getDefaultExecutor();
dve.setObjectMapper(objectMapper);
dve.setMetadata(getMetadata());
if(depositionVersion.getDOIURL()!=null) {
boolean updateEnabled = false;
Boolean update = depositionVersion.getUpdate();
try {
String updatedDeposit = Config.getProperties().getProperty("update_deposit");
updateEnabled = Boolean.valueOf(updatedDeposit);
}catch (Exception e) {
updateEnabled = false;
if(update==null) {
update = depositionVersion.getDeposition().getDefaultUpdate();
}
logger.trace("Deposition update {}", updateEnabled ? "enabled" : "disabled");
if(updateEnabled) {
if(update) {
logger.info("Going to update deposition {} {}.",
name , depositionVersion.getVersion());
dve.setDOIURL(depositionVersion.getDOIURL());

10
src/main/java/org/gcube/common/deposition/executor/DepositionVersionExecutor.java
View File

@ -4,6 +4,7 @@ import java.net.URL;
import java.util.List;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.common.deposition.executor.zenodo.ZenodoDepositionVersionExecutor;
import org.gcube.common.deposition.model.DepositionFile;
@ -16,10 +17,19 @@ public abstract class DepositionVersionExecutor {
return new ZenodoDepositionVersionExecutor();
}
protected ObjectMapper objectMapper;
protected JsonNode metadata;
protected URL doiURL;
protected List<DepositionFile> depositionFiles;
public ObjectMapper getObjectMapper() {
return objectMapper;
}
public void setObjectMapper(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public JsonNode getMetadata() {
return metadata;
}

235
src/main/java/org/gcube/common/deposition/executor/zenodo/ZenodoDepositionVersionExecutor.java
View File

@ -1,16 +1,34 @@
package org.gcube.common.deposition.executor.zenodo;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.common.deposition.config.Config;
import org.gcube.common.deposition.executor.DepositionVersionExecutor;
import org.gcube.common.deposition.model.DepositionFile;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -21,13 +39,43 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
private static final Logger logger = LoggerFactory.getLogger(ZenodoDepositionVersionExecutor.class);
public static final String GUCBE_ZENODO_SOFTWARE_DEPOSIT = "gCubeSoftwareDeposit";
public static final String METADATA_FIELD_NAME = "metadata";
public static final String ZENODO_DOI_URL_BASE_PATH = "https://doi.org/10.5072/zenodo.";
public static final String DEPOSITIONS_COLLECTION_PATH = "/api/deposit/depositions";
public static final String DEPOSITION_PATH = DEPOSITIONS_COLLECTION_PATH + ":id";
public static final String DEPOSTION_FILES_PATH = DEPOSITION_PATH + "/files";
public static final String DEPOSTION_NEW_VERSION_PATH = DEPOSITION_PATH + "/actions/newversion";
public static final String DEPOSTION_PUBLISH_PATH = DEPOSITION_PATH + "/actions/publish";
protected URL zenodoBaseURL;
protected String accessToken;
protected String zenodoID;
protected JsonNode response;
protected String getZenodoIDFromDOIURL(URL doiURL) {
return doiURL.toString().replace(ZENODO_DOI_URL_BASE_PATH, "");
}
protected Map<String, String> getAccessTokenQueryParamters() {
Map<String, String> map = new HashMap<>();
map.put("access_token", accessToken);
return map;
}
public ZenodoDepositionVersionExecutor() {
try {
this.zenodoBaseURL = new URL(Config.getProperties().getProperty("zenodo_base_url"));
this.accessToken = Config.getProperties().getProperty("zenodo_access_token");
if(doiURL!=null) {
}
}catch (Exception e) {
throw new RuntimeException(e);
}
@ -52,6 +100,73 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
return file;
}
protected void addFilesToDeposition(List<File> files ) throws Exception {
String depositID = getZenodoIDFromDOIURL(doiURL);
String newFilePath = DEPOSTION_FILES_PATH.replace(":id", depositID);
URL url = new URL(zenodoBaseURL, newFilePath);
for(File file : files) {
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost uploadFile = new HttpPost(url.toURI());
uploadFile.addHeader("access_token", accessToken);
uploadFile.addHeader("Accept", "application/json");
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("name", file.getName(), ContentType.TEXT_PLAIN);
builder.addBinaryBody("file", new FileInputStream(file),
ContentType.APPLICATION_OCTET_STREAM,
file.getName()
);
HttpEntity multipart = builder.build();
uploadFile.setEntity(multipart);
CloseableHttpResponse response = httpClient.execute(uploadFile);
HttpEntity responseEntity = response.getEntity();
StringBuilder result = getStringBuilder(responseEntity.getContent());
String res = result.toString();
logger.debug("Created file from {} response is {}", file.getAbsolutePath(), res);
// JsonNode jsonNode = objectMapper.readTree(res);
}
}
protected void updateMetadata() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
String conceptID = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", conceptID));
ObjectNode body = objectMapper.createObjectNode();
body.set(METADATA_FIELD_NAME, metadata);
HttpURLConnection httpURLConnection = gxHTTPStringRequest.put(objectMapper.writeValueAsString(body));
JsonNode jsonNode = getResponse(httpURLConnection);
}
protected void publish() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
String id = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSTION_PUBLISH_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
JsonNode jsonNode = getResponse(httpURLConnection);
}
protected void finalize() throws Exception {
List<File> files = new ArrayList<>();
for(DepositionFile df : depositionFiles) {
@ -61,10 +176,14 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
// TODO Add depositionFiles
// TODO Update deposit metadata
//Add depositionFiles
addFilesToDeposition(files);
// TODO Publish the version
//Update deposit metadata
updateMetadata();
// Publish the version
publish();
for(File file : files) {
if(!file.exists()) {
@ -78,9 +197,79 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
}
}
protected StringBuilder getStringBuilder(InputStream inputStream) throws IOException {
StringBuilder result = new StringBuilder();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
String line;
while ((line = reader.readLine()) != null) {
result.append(line);
}
}
return result;
}
public JsonNode getResponse(HttpURLConnection connection) throws Exception {
try {
int responseCode = connection.getResponseCode();
String responseMessage = connection.getResponseMessage();
logger.trace("Response {} {}", responseCode, responseMessage);
if(connection.getRequestMethod().compareTo("HEAD")==0) {
if(responseCode == HttpURLConnection.HTTP_NO_CONTENT) {
return null;
}
if(responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
throw new RuntimeException(responseCode + " " + responseMessage);
}
if(responseCode == HttpURLConnection.HTTP_FORBIDDEN) {
throw new RuntimeException(responseCode + " " + responseMessage);
}
}
if(responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) {
InputStream inputStream = connection.getErrorStream();
StringBuilder result = getStringBuilder(inputStream);
String res = result.toString();
throw new RuntimeException(res);
}
StringBuilder result = getStringBuilder(connection.getInputStream());
String res = result.toString();
logger.trace("Server returned content : {}", res);
return objectMapper.readTree(res);
} finally {
connection.disconnect();
}
}
@Override
public void create() throws Exception {
// TODO New deposit on Zenodo
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
gxHTTPStringRequest.path(DEPOSITIONS_COLLECTION_PATH);
ObjectNode body = objectMapper.createObjectNode();
body.set(METADATA_FIELD_NAME, metadata);
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post(objectMapper.writeValueAsString(body));
response = getResponse(httpURLConnection);
doiURL = new URL(ZENODO_DOI_URL_BASE_PATH + response.get("conceptrecid").asText());
finalize();
}
@ -89,10 +278,44 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
// TODO edit a published version
}
/**
* Remove previous depositionFiles
* @throws Exception
*/
protected void deletePreviousFiles() throws Exception {
ArrayNode files = (ArrayNode) response.get("files");
for(int i=0; i<files.size(); i++) {
ObjectNode file = (ObjectNode) files.get(i);
String fileURLString = file.get("links").get("self").asText();
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(fileURLString);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
HttpURLConnection httpURLConnection = gxHTTPStringRequest.delete();
getResponse(httpURLConnection);
}
}
@Override
public void newVersion() throws Exception {
// TODO New version on Zenodo from previous using the id in the DOI
// TODO Remove previous depositionFiles
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
// Creating the new version starting form doiURL which contains conceptID DOI
String id = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSTION_NEW_VERSION_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
response = getResponse(httpURLConnection);
// The doi of this created new version
doiURL = new URL(response.get("doi_url").asText());
// Remove previous depositionFiles
deletePreviousFiles();
finalize();
}

23
src/main/java/org/gcube/common/deposition/model/Deposition.java
View File

@ -13,6 +13,9 @@ public class Deposition {
@JsonProperty(value = "name")
protected String name;
@JsonProperty(value = "default_update")
protected Boolean defaultUpdate;
@JsonProperty(value = "default_files")
protected List<DepositionFile> defaultFiles;
@ -32,34 +35,22 @@ public class Deposition {
return name;
}
public void setName(String name) {
this.name = name;
public Boolean getDefaultUpdate() {
return defaultUpdate;
}
public List<DepositionFile> getDefaultFiles() {
return defaultFiles;
}
public void setDefaultFiles(List<DepositionFile> defaultFiles) {
this.defaultFiles = defaultFiles;
}
public String getDefaultCodeLocation() {
return defaultCodeLocation;
}
public void setDefaultCodeLocation(String defaultCodeLocation) {
this.defaultCodeLocation= defaultCodeLocation;
}
public String getCodeLocationAdditionalDescription() {
return codeLocationAdditionalDescription;
}
public void setCodeLocationAdditionalDescription(String codeLocationAdditionalDescription) {
this.codeLocationAdditionalDescription = codeLocationAdditionalDescription;
}
public List<DepositionVersion> getDepositionVersions() {
return depositionVersions;
}
@ -68,8 +59,4 @@ public class Deposition {
return metadata;
}
public void setMetadata(JsonNode metadata) {
this.metadata = metadata;
}
}

11
src/main/java/org/gcube/common/deposition/model/DepositionVersion.java
View File

@ -59,6 +59,9 @@ public class DepositionVersion {
@JsonProperty(value = "doi_url")
protected URL doiURL;
@JsonProperty(value = "update")
protected Boolean update;
public DepositionVersion() {
this.noCodeLocation = false;
@ -183,6 +186,14 @@ public class DepositionVersion {
this.doiURL = null;
}
}
public Boolean getUpdate() {
return update;
}
public void setUpdate(Boolean update) {
this.update = update;
}

4
src/test/resources/zenodo-deposit.json
View File

@ -1,5 +1,6 @@
{
"name": "gcat",
"default_update": false,
"default_files":
[
{
@ -31,7 +32,8 @@
"release_ticket": null,
"code_location": null,
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url": "https://doi.org/10.5072/zenodo.1139446"
"doi_url": "https://doi.org/10.5072/zenodo.1139446",
"update": true
},
{
"version": "1.1.0",