gCubeSystem
/
software-versions-processor-lib
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

First working solution

This commit is contained in:
Luca Frosini 2023-01-04 17:28:20 +01:00
parent 29431f67ac
commit 7049f38ba1
6 changed files with 182 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pom.xml
View File

@ -49,23 +49,30 @@
<groupId>org.gcube.common</groupId>
<artifactId>gxHTTP</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-multipart</artifactId>
</dependency>
<!--
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.6</version>
</dependency>
-->
<!-- Test libraries -->
<dependency>

2
src/main/java/org/gcube/common/deposition/ElaborateDeposition.java
View File

@ -55,7 +55,7 @@ public class ElaborateDeposition {
previous = depositionVersion;
Thread.sleep(TimeUnit.SECONDS.toMillis(3));
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
}
// TODO Save content to file

2
src/main/java/org/gcube/common/deposition/ElaborateDepositionVersion.java
View File

@ -12,7 +12,6 @@ import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
import org.gcube.com.fasterxml.jackson.databind.node.TextNode;
import org.gcube.common.deposition.config.Config;
import org.gcube.common.deposition.executor.DepositionVersionExecutor;
import org.gcube.common.deposition.model.Deposition;
import org.gcube.common.deposition.model.DepositionFile;
@ -178,6 +177,7 @@ public class ElaborateDepositionVersion {
logger.trace("DepositionVersion to be managed is:{}", objectMapper.writeValueAsString(depositionVersion));
DepositionVersionExecutor dve = DepositionVersionExecutor.getDefaultExecutor();
dve.setDepositionVersion(depositionVersion);
dve.setObjectMapper(objectMapper);
dve.setMetadata(getMetadata());

10
src/main/java/org/gcube/common/deposition/executor/DepositionVersionExecutor.java
View File

@ -7,6 +7,7 @@ import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.ObjectMapper;
import org.gcube.common.deposition.executor.zenodo.ZenodoDepositionVersionExecutor;
import org.gcube.common.deposition.model.DepositionFile;
import org.gcube.common.deposition.model.DepositionVersion;
/**
* @author Luca Frosini (ISTI - CNR)
@ -17,11 +18,20 @@ public abstract class DepositionVersionExecutor {
return new ZenodoDepositionVersionExecutor();
}
protected DepositionVersion depositionVersion;
protected ObjectMapper objectMapper;
protected JsonNode metadata;
protected URL doiURL;
protected List<DepositionFile> depositionFiles;
public DepositionVersion getDepositionVersion() {
return depositionVersion;
}
public void setDepositionVersion(DepositionVersion depositionVersion) {
this.depositionVersion = depositionVersion;
}
public ObjectMapper getObjectMapper() {
return objectMapper;
}

173
src/main/java/org/gcube/common/deposition/executor/zenodo/ZenodoDepositionVersionExecutor.java
View File

@ -2,26 +2,28 @@ package org.gcube.common.deposition.executor.zenodo;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.gcube.com.fasterxml.jackson.databind.JsonNode;
import org.gcube.com.fasterxml.jackson.databind.node.ArrayNode;
import org.gcube.com.fasterxml.jackson.databind.node.ObjectNode;
@ -29,6 +31,10 @@ import org.gcube.common.deposition.config.Config;
import org.gcube.common.deposition.executor.DepositionVersionExecutor;
import org.gcube.common.deposition.model.DepositionFile;
import org.gcube.common.gxhttp.request.GXHTTPStringRequest;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.media.multipart.file.FileDataBodyPart;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -46,13 +52,16 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
public static final String ZENODO_DOI_URL_BASE_PATH = "https://doi.org/10.5072/zenodo.";
public static final String DEPOSITIONS_COLLECTION_PATH = "/api/deposit/depositions";
public static final String DEPOSITION_PATH = DEPOSITIONS_COLLECTION_PATH + ":id";
public static final String DEPOSITION_PATH = DEPOSITIONS_COLLECTION_PATH + "/:id";
public static final String RECORD_PATH = "/api/records/:id";
public static final String DEPOSTION_FILES_PATH = DEPOSITION_PATH + "/files";
public static final String DEPOSTION_NEW_VERSION_PATH = DEPOSITION_PATH + "/actions/newversion";
public static final String DEPOSTION_EDIT_PATH = DEPOSITION_PATH + "/actions/edit";
public static final String DEPOSTION_PUBLISH_PATH = DEPOSITION_PATH + "/actions/publish";
protected URL zenodoBaseURL;
protected String accessToken;
@ -60,7 +69,11 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
protected JsonNode response;
protected String getZenodoIDFromDOIURL(URL doiURL) {
return doiURL.toString().replace(ZENODO_DOI_URL_BASE_PATH, "");
return getZenodoIDFromDOIURL(doiURL.toString());
}
protected String getZenodoIDFromDOIURL(String doiURL) {
return doiURL.replace(ZENODO_DOI_URL_BASE_PATH, "");
}
protected Map<String, String> getAccessTokenQueryParamters() {
@ -82,14 +95,15 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
}
protected File downloadFile(DepositionFile df) throws IOException {
// Path path = Paths.get(df.getDesiredName());
// try (InputStream inputStream = df.getURL().openStream()) {
// Files.copy(inputStream, path, StandardCopyOption.REPLACE_EXISTING);
// }
File file = new File(df.getDesiredName());
Path path = Paths.get(df.getDesiredName());
try (InputStream inputStream = df.getURL().openStream()) {
Files.copy(inputStream, path, StandardCopyOption.REPLACE_EXISTING);
}
/* Uses apache common-io */
FileUtils.copyURLToFile(df.getURL(), file);
// FileUtils.copyURLToFile(df.getURL(), file);
if(!file.exists()) {
throw new RuntimeException(file.getAbsolutePath() + " does not exist");
@ -106,54 +120,49 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
URL url = new URL(zenodoBaseURL, newFilePath);
for(File file : files) {
Client client = ClientBuilder.newClient()
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);;
client.register(MultiPartFeature.class);
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost uploadFile = new HttpPost(url.toURI());
uploadFile.addHeader("access_token", accessToken);
uploadFile.addHeader("Accept", "application/json");
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("name", file.getName(), ContentType.TEXT_PLAIN);
builder.addBinaryBody("file", new FileInputStream(file),
ContentType.APPLICATION_OCTET_STREAM,
file.getName()
);
FormDataMultiPart multi=new FormDataMultiPart();
FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file", file, MediaType.APPLICATION_OCTET_STREAM_TYPE);
multi.field("name", file.getName());
multi.bodyPart(fileDataBodyPart);
HttpEntity multipart = builder.build();
uploadFile.setEntity(multipart);
Response response = client.target(url.toURI().toString())
.queryParam("access_token", accessToken)
.request("application/json")
.post(Entity.entity(multi,multi.getMediaType()));
CloseableHttpResponse response = httpClient.execute(uploadFile);
HttpEntity responseEntity = response.getEntity();
StringBuilder result = getStringBuilder(responseEntity.getContent());
String res = result.toString();
logger.debug("Created file from {} response is {}", file.getAbsolutePath(), res);
// JsonNode jsonNode = objectMapper.readTree(res);
int statusCode = response.getStatus();
if(statusCode>400) {
throw new RuntimeException("Error while uploading file " + file.getAbsolutePath());
}
}
}
protected void updateMetadata() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
String conceptID = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", conceptID));
String id = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", id));
ObjectNode body = objectMapper.createObjectNode();
body.set(METADATA_FIELD_NAME, metadata);
HttpURLConnection httpURLConnection = gxHTTPStringRequest.put(objectMapper.writeValueAsString(body));
JsonNode jsonNode = getResponse(httpURLConnection);
getResponse(httpURLConnection);
}
protected void publish() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
@ -163,8 +172,7 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
gxHTTPStringRequest.path(DEPOSTION_PUBLISH_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
JsonNode jsonNode = getResponse(httpURLConnection);
getResponse(httpURLConnection);
}
protected void finalize() throws Exception {
@ -172,10 +180,9 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
for(DepositionFile df : depositionFiles) {
File file = downloadFile(df);
files.add(file);
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
}
Thread.sleep(TimeUnit.SECONDS.toMillis(5));
//Add depositionFiles
addFilesToDeposition(files);
@ -218,7 +225,6 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
logger.trace("Response {} {}", responseCode, responseMessage);
if(connection.getRequestMethod().compareTo("HEAD")==0) {
if(responseCode == HttpURLConnection.HTTP_NO_CONTENT) {
return null;
}
@ -226,10 +232,10 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
if(responseCode == HttpURLConnection.HTTP_NOT_FOUND) {
throw new RuntimeException(responseCode + " " + responseMessage);
}
if(responseCode == HttpURLConnection.HTTP_FORBIDDEN) {
throw new RuntimeException(responseCode + " " + responseMessage);
}
}
if(responseCode >= HttpURLConnection.HTTP_BAD_REQUEST) {
@ -253,9 +259,14 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
}
}
protected URL createZenodoDOIURLFromID(String id) throws MalformedURLException {
return new URL(ZENODO_DOI_URL_BASE_PATH + id);
}
@Override
public void create() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
@ -268,14 +279,32 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post(objectMapper.writeValueAsString(body));
response = getResponse(httpURLConnection);
doiURL = new URL(ZENODO_DOI_URL_BASE_PATH + response.get("conceptrecid").asText());
URL conceptDOIURL = createZenodoDOIURLFromID(response.get("conceptrecid").asText());
depositionVersion.setConceptDOIURL(conceptDOIURL);
doiURL = new URL(ZENODO_DOI_URL_BASE_PATH + response.get("id").asText());
depositionVersion.setDOIURL(doiURL);
finalize();
}
@Override
public void update() throws Exception {
// TODO edit a published version
// Enabled deposition edit
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Accept", "application/json");
String id = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSTION_EDIT_PATH.replace(":id", id));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
getResponse(httpURLConnection);
//Update deposit metadata
updateMetadata();
// Publish the version
publish();
}
/**
@ -288,6 +317,7 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
ObjectNode file = (ObjectNode) files.get(i);
String fileURLString = file.get("links").get("self").asText();
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(fileURLString);
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
HttpURLConnection httpURLConnection = gxHTTPStringRequest.delete();
@ -298,20 +328,55 @@ public class ZenodoDepositionVersionExecutor extends DepositionVersionExecutor {
@Override
public void newVersion() throws Exception {
GXHTTPStringRequest gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
// Creating the new version starting form doiURL which contains conceptID DOI
String id = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(DEPOSTION_NEW_VERSION_PATH.replace(":id", id));
String conceptID = getZenodoIDFromDOIURL(doiURL);
gxHTTPStringRequest.path(RECORD_PATH.replace(":id", conceptID));
HttpURLConnection httpURLConnection = gxHTTPStringRequest.post();
HttpURLConnection httpURLConnection = gxHTTPStringRequest.get();
JsonNode jsonNode = getResponse(httpURLConnection);
String latestDOI = jsonNode.get("links").get("doi").asText();
String previousVersionDOI = depositionVersion.getPrevious().getDOIURL().toString();
if(previousVersionDOI.compareTo(latestDOI)!=0) {
logger.error("Zenodo obtained latest DOI {} != {} DOI form previous version", latestDOI, previousVersionDOI);
throw new RuntimeException("It seems that your json is not up to date with Zenodo.");
}
gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Content-Type", "application/json");
gxHTTPStringRequest.header("Accept", "application/json");
// Creating the new version starting form doiURL which contains conceptID DOI
String latestID = getZenodoIDFromDOIURL(latestDOI);
gxHTTPStringRequest.path(DEPOSTION_NEW_VERSION_PATH.replace(":id", latestID));
httpURLConnection = gxHTTPStringRequest.post();
jsonNode = getResponse(httpURLConnection);
String draftURL = jsonNode.get("links").get("latest_draft").asText();
String draftID = draftURL.replace(zenodoBaseURL.toString() + DEPOSITIONS_COLLECTION_PATH + "/", "");
gxHTTPStringRequest = GXHTTPStringRequest.newRequest(zenodoBaseURL.toString());
gxHTTPStringRequest.isExternalCall(true);
gxHTTPStringRequest.from(GUCBE_ZENODO_SOFTWARE_DEPOSIT);
gxHTTPStringRequest.queryParams(getAccessTokenQueryParamters());
gxHTTPStringRequest.header("Accept", "application/json");
gxHTTPStringRequest.path(DEPOSITION_PATH.replace(":id", draftID));
httpURLConnection = gxHTTPStringRequest.get();
response = getResponse(httpURLConnection);
// The doi of this created new version
doiURL = new URL(response.get("doi_url").asText());
depositionVersion.setDOIURL(doiURL);
// Remove previous depositionFiles
deletePreviousFiles();

24
src/test/resources/zenodo-deposit.json
View File

@ -32,8 +32,7 @@
"release_ticket": null,
"code_location": null,
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url": "https://doi.org/10.5072/zenodo.1139446",
"update": true
"doi_url": "https://doi.org/10.5072/zenodo.1139446"
},
{
"version": "1.1.0",
@ -85,49 +84,56 @@
],
"release_ticket": "https://support.d4science.org/issues/16743",
"code_location": null,
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url": "https://doi.org/10.5072/zenodo.1143572"
},
{
"version": "1.4.0",
"date": "2019-11-20",
"gcube_release_version": "4.15.0",
"release_ticket": "https://support.d4science.org/issues/17294",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url" : "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143583"
},
{
"version": "1.4.1",
"date": "2019-12-20",
"gcube_release_version": "4.18.0",
"release_ticket": "https://support.d4science.org/issues/18335",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143585"
},
{
"version": "1.4.2",
"date": "2020-02-14",
"gcube_release_version": "4.20.0",
"release_ticket": "https://support.d4science.org/issues/18507",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143586"
},
{
"version": "1.4.3",
"date": "2020-06-16",
"gcube_release_version": "4.23.0",
"release_ticket": "https://support.d4science.org/issues/19322",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143587"
},
{
"version": "1.4.4",
"date": "2021-02-24",
"gcube_release_version": "5.0.0",
"release_ticket": "https://support.d4science.org/issues/20648",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143589"
},
{
"version": "1.4.5",
"date": "2021-05-31",
"gcube_release_version": "5.1.0",
"release_ticket": "https://support.d4science.org/issues/20920",
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445"
"concept_doi_url": "https://doi.org/10.5072/zenodo.1139445",
"doi_url" : "https://doi.org/10.5072/zenodo.1143590"
}
],
"metadata": {