Merge pull request 'fix_26166' (!6) from fix_26166 into master

Reviewed-on: #6
This commit is contained in:
Francesco Mangiacrapa 2023-12-21 09:00:35 +01:00
commit 0e98b7c266
26 changed files with 263 additions and 44 deletions

View File

@ -1,6 +1,11 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8

View File

@ -1,6 +1,10 @@
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
# Changelog for org.gcube.data.publishing.ckan2zenodo-library
## [v1.0.4-SNAPSHOT] 2022-12-13
- Bug fixing `publication_date=null` [#26166]
## [v1.0.3] 2023-03-28
- Extensions evaluated from URL [#22889](https://support.d4science.org/issues/22889)

View File

@ -4,11 +4,11 @@
<parent>
<groupId>org.gcube.tools</groupId>
<artifactId>maven-parent</artifactId>
<version>1.1.0</version>
<version>1.2.0</version>
</parent>
<groupId>org.gcube.data.publishing</groupId>
<artifactId>ckan2zenodo-library</artifactId>
<version>1.0.3</version>
<version>1.0.4-SNAPSHOT</version>
<name>CKAN 2 Zenodo Library</name>
<description>Library to publish d4science CKAN items into Zenodo</description>

View File

@ -7,7 +7,6 @@ import java.util.concurrent.Future;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.UploadPolicy;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.GcatException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;

View File

@ -177,7 +177,16 @@ public class Ckan2ZenodoImpl implements Ckan2Zenodo{
if(!skipErrors) throw e;
else tr=new Translator();
}
return tr.transform(desc, toUpdate);
/* original
* return tr.transform(desc, toUpdate);
*/
//Added logs by Francesco M.
ZenodoDeposition deposition = tr.transform(desc, toUpdate);
log.debug("returning from translate: {}",deposition);
return deposition;
}
}

View File

@ -2,7 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
@ -52,6 +51,11 @@ public class Fixer {
}
/**
* It seems never used to me - by Francesco M.
* @param toFix
* @return
*/
public static String fixSending(String toFix) {
DocumentContext ctx=JsonPath.using(Parsing.JSON_PATH_ALWAYS_LIST_CONFIG).parse(toFix);
DocumentContext pathCtx=JsonPath.using(Parsing.JSON_PATH_PATHS_CONFIGURATION).parse(toFix);
@ -85,6 +89,12 @@ public class Fixer {
return valuesCtx;
}
/**
* Updated by Francesco M., see #26166
* @param toFix
* @return
* @throws Exception
*/
public static final String fixIncomingDateString(String toFix) throws Exception {
TemporalAccessor accessor=INCOMING_FORMATTER.parse(toFix);
@ -93,7 +103,11 @@ public class Fixer {
zTime = LocalDateTime.from(accessor).atZone(ZoneOffset.UTC);
else {
LocalDate date=LocalDate.from(accessor);
zTime=LocalDateTime.of(date, LocalTime.of(0, 0, 0, 0)).atZone(ZoneOffset.UTC);
//original code
//zTime=LocalDateTime.of(date, LocalTime.of(0, 0, 0, 0)).atZone(ZoneOffset.UTC);
//new code by Francesco M. returning a date without the Time
return date.toString();
}
return zTime.format(INTERNAL_FORMATTER);

View File

@ -4,7 +4,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;

View File

@ -7,9 +7,9 @@ import javax.xml.parsers.ParserConfigurationException;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.data.publishing.ckan2zenodo.commons.IS;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import lombok.Synchronized;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
public class TransformerManager {

View File

@ -1,7 +1,10 @@
package org.gcube.data.publishing.ckan2zenodo.clients;
import java.nio.file.Files;
import java.util.*;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;

View File

@ -59,7 +59,9 @@ public class Zenodo {
private static final String ACCESS_TOKEN="access_token";
private static ObjectMapper mapper = new ObjectMapper();
//private static ObjectMapper mapper = new ObjectMapper();
//Updated by Francesco M.
public static final ObjectMapper mapper = new ObjectMapper();
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,false);
@ -223,16 +225,20 @@ public class Zenodo {
}
private ZenodoDeposition updateMetadata(Integer depositionId,DepositionMetadata meta) throws ZenodoException {
//Added logs by Francesco M.
try{
String serialized="{\"metadata\":"+Fixer.fixIncoming(mapper.writeValueAsString(meta))+"}";
log.debug("updateMetadata - serialized: {}",serialized);
try {
Entity<String> jsonEnt = Entity.json(serialized);
Response resp = getWebClient().target(credentials.getBaseUrl()).
path(DEPOSITION_BASE_URL).path(depositionId+"").
queryParam(ACCESS_TOKEN, credentials.getKey()).request(CONTENT_TYPE)
.put(Entity.json(serialized));
.put(jsonEnt);
return check(resp,ZenodoDeposition.class);
}catch(Throwable t) {
log.debug("Error while tryin to update "+serialized);
log.debug("Error while trying to update "+serialized);
throw t;
}
}catch(JsonProcessingException e) {

View File

@ -4,7 +4,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -16,7 +15,6 @@ import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mapping;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.Mappings;
import org.gcube.data.publishing.ckan2zenodo.model.parsing.ResourceFilter;
import org.gcube.resources.discovery.client.api.DiscoveryClient;

View File

@ -7,7 +7,6 @@ import java.util.List;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;

View File

@ -9,11 +9,12 @@ import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.*;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

View File

@ -1,13 +1,10 @@
package org.gcube.data.publishing.ckan2zenodo.model.report;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
@Data
public class EnvironmentReport {

View File

@ -2,7 +2,9 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
public class Commons {
public static final String ISO_DATE_PATTERN="yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ";
public static final String ISO_DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ";
// Added by Francesco M. #26166
public static final String ISO_DATE_PATTERN_WITHOUT_TIME = "yyyy-MM-dd";
}

View File

@ -2,7 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@Data
@NoArgsConstructor

View File

@ -2,8 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;

View File

@ -2,8 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@NoArgsConstructor

View File

@ -17,11 +17,13 @@ public class DateInterval {
public static enum Type{
Collected, Valid, Withdrawn
}
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
//Updated by Francesco M. #26166
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN_WITHOUT_TIME)
@JsonDeserialize(using = DateParser.class)
private Date start;
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
//Updated by Francesco M. #26166
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN_WITHOUT_TIME)
@JsonDeserialize(using = DateParser.class)
private Date end;
// @NonNull

View File

@ -7,8 +7,6 @@ import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@NoArgsConstructor
@ -68,8 +66,8 @@ public class DepositionMetadata {
private UploadType upload_type;
private PublicationType publication_type;
private ImageType image_type;
// @NonNull
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
// @NonNull - Updated by Francesco M.
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN_WITHOUT_TIME)
private Date publication_date;
// @NonNull
private String title;
@ -80,7 +78,7 @@ public class DepositionMetadata {
// @NonNull
private AccessRights access_right;
private String license; // TODO ENUM https://licenses.opendefinition.org/licenses/groups/all.json
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN)
@JsonFormat(pattern = Commons.ISO_DATE_PATTERN_WITHOUT_TIME)
private Date embargo_date;
private String access_conditions; // TODO HTML
private String doi;

View File

@ -2,7 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@Data
@NoArgsConstructor

View File

@ -2,7 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@Data
@NoArgsConstructor

View File

@ -2,8 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@Data
@NoArgsConstructor

View File

@ -2,7 +2,6 @@ package org.gcube.data.publishing.ckan2zenodo.model.zenodo;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@Data
@NoArgsConstructor

View File

@ -4,12 +4,15 @@ import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.ZenodoCredentials;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@ -74,4 +77,5 @@ public class TestCommons {
throw new Exception("Errors with json file "+jsonFile,t);
}
}
}

View File

@ -0,0 +1,189 @@
package org.gcube.data.publishing.ckan2zenodo;
import java.io.IOException;
import java.time.Instant;
import java.util.Date;
import javax.ws.rs.client.Entity;
import org.gcube.data.publishing.ckan2zenodo.clients.Zenodo;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.DepositionMetadata;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
/**
* The Class Test_BugFix_26166.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Dec 20, 2023
*/
@Slf4j
public class Test_BugFix_26166 {
@Test
public void testDate(){
ZenodoDeposition zenodo = new ZenodoDeposition();
zenodo.setCreated(Date.from(Instant.now()));
zenodo.setModified(Date.from(Instant.now()));
DepositionMetadata metadata = new DepositionMetadata();
metadata.setPublication_date(Date.from(Instant.now()));
metadata.setEmbargo_date(Date.from(Instant.now()));
zenodo.setMetadata(metadata);
ObjectMapper mapper = Zenodo.mapper;
try {
String serialized = "{\"metadata\":" + Fixer.fixIncoming(mapper.writeValueAsString(zenodo)) + "}";
log.debug("updateMetadata - serialized: {}", serialized);
try {
Entity<String> jsonEnt = Entity.json(serialized);
log.debug("updateMetadata - jsonEntity serialized: {}", jsonEnt.getEntity());
} catch (Throwable t) {
log.debug("Error while tryin to update " + serialized);
throw t;
}
} catch (JsonProcessingException e) {
log.debug("Error while parsing " + metadata, e);
e.printStackTrace();
}
}
@Test
public void testReadDepositionFromZenodo(){
String depoisitionString = " {\n"
+ " \"created\": \"2023-12-14T11:33:09.650134+00:00\",\n"
+ " \"modified\": \"2023-12-14T11:33:09.700320+00:00\",\n"
+ " \"id\": 10377042,\n"
+ " \"conceptrecid\": \"10377041\",\n"
+ " \"metadata\": {\n"
+ " \"title\": \"Blue-Cloud 2026 - D4.2 VLabs Implementation Guidelines\",\n"
+ " \"publication_date\": \"2023-12-14\",\n"
+ " \"description\": \"This deliverable provides guidelines on how to implement the Virtual \\nLaboratories (VLabs) in the Blue-Cloud Virtual Research Environment (VRE). \\nFirst a summary of the activities of WP4 is presented, followed by an \\noverview of the VRE services, that offer a comprehensive array of features, \\nfostering collaboration, facilitating data analytics, enabling result \\ndissemination, and ensuring seamless integration with external systems. In \\nfact, the VRE caters to the entirety of the research lifecycle.\",\n"
+ " \"access_right\": \"open\",\n"
+ " \"creators\": [\n"
+ " {\n"
+ " \"name\": \"Assante, Massimiliano\",\n"
+ " \"affiliation\": \"National Research Council of Italy - Institute of Information Science and Technologies\",\n"
+ " \"orcid\": \"0000-0002-3761-1492\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Martin-Cabrera\",\n"
+ " \"affiliation\": \"Flanders Marine Institute (VLIZ)\",\n"
+ " \"orcid\": \"0000-0003-3293-5612\"\n"
+ " }\n"
+ " ],\n"
+ " \"contributors\": [\n"
+ " {\n"
+ " \"name\": \"Vitorino, Joao\",\n"
+ " \"affiliation\": null,\n"
+ " \"type\": \"Other\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Troupin, Charles, ctroupin@uliege.be, orcid.org/\",\n"
+ " \"affiliation\": \"Université de Liège\",\n"
+ " \"type\": \"Other\",\n"
+ " \"orcid\": \"0000-0002-0265-1021\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Pint, Steven\",\n"
+ " \"affiliation\": \"Flanders Marine Institute (VLIZ)\",\n"
+ " \"type\": \"Other\",\n"
+ " \"orcid\": \"0000-0001-5530-6785\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Palermo, Francesco\",\n"
+ " \"affiliation\": \"Centro Euro-Mediterraneo sui Cambiamenti Climatici (CMCC)\",\n"
+ " \"type\": \"Other\",\n"
+ " \"orcid\": \"0000-0002-0463-4655\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Grasset, Bastien\",\n"
+ " \"affiliation\": \"French National Research Institute for Sustainable Development (IRD)\",\n"
+ " \"type\": \"Other\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Barth, Alexander\",\n"
+ " \"affiliation\": \"Université de Liège\",\n"
+ " \"type\": \"Other\",\n"
+ " \"orcid\": \"0000-0003-2952-5997\"\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"Barde, Julien\",\n"
+ " \"affiliation\": \"French National Research Institute for Sustainable Development (IRD)\",\n"
+ " \"type\": \"Other\",\n"
+ " \"orcid\": \"0000-0002-3519-6141\"\n"
+ " }\n"
+ " ],\n"
+ " \"keywords\": [\n"
+ " \"VLabs\",\n"
+ " \"VRE\"\n"
+ " ],\n"
+ " \"related_identifiers\": [\n"
+ " {\n"
+ " \"identifier\": \"https://data.d4science.org/ctlg/Blue-Cloud2026Project/d4_2_vlabs_implementation_guidelines\",\n"
+ " \"relation\": \"isCompiledBy\",\n"
+ " \"scheme\": \"url\"\n"
+ " }\n"
+ " ],\n"
+ " \"version\": \"1\",\n"
+ " \"grants\": [\n"
+ " {\n"
+ " \"id\": \"10.13039/501100000780::101094227\"\n"
+ " }\n"
+ " ],\n"
+ " \"license\": \"cc-by-4.0\",\n"
+ " \"imprint_publisher\": \"Zenodo\",\n"
+ " \"communities\": [\n"
+ " {\n"
+ " \"identifier\": \"bluecloud\"\n"
+ " }\n"
+ " ],\n"
+ " \"upload_type\": \"publication\",\n"
+ " \"publication_type\": \"deliverable\",\n"
+ " \"prereserve_doi\": {\n"
+ " \"doi\": \"10.5281/zenodo.10377042\",\n"
+ " \"recid\": 10377042\n"
+ " }\n"
+ " },\n"
+ " \"title\": \"Blue-Cloud 2026 - D4.2 VLabs Implementation Guidelines\",\n"
+ " \"links\": {\n"
+ " \"self\": \"https://zenodo.org/api/records/10377042\",\n"
+ " \"html\": \"https://zenodo.org/records/10377042\",\n"
+ " \"badge\": \"https://zenodo.org/badge/doi/.svg\",\n"
+ " \"files\": \"https://zenodo.org/api/records/10377042/files\",\n"
+ " \"latest_draft\": \"https://zenodo.org/api/deposit/depositions/10377042\",\n"
+ " \"latest_draft_html\": \"https://zenodo.org/deposit/10377042\",\n"
+ " \"publish\": \"https://zenodo.org/api/deposit/depositions/10377042/actions/publish\",\n"
+ " \"edit\": \"https://zenodo.org/api/deposit/depositions/10377042/actions/edit\",\n"
+ " \"discard\": \"https://zenodo.org/api/deposit/depositions/10377042/actions/discard\",\n"
+ " \"newversion\": \"https://zenodo.org/api/deposit/depositions/10377042/actions/newversion\",\n"
+ " \"registerconceptdoi\": \"https://zenodo.org/api/deposit/depositions/10377042/actions/registerconceptdoi\"\n"
+ " },\n"
+ " \"record_id\": 10377042,\n"
+ " \"owner\": 82231,\n"
+ " \"files\": [],\n"
+ " \"state\": \"unsubmitted\",\n"
+ " \"submitted\": false\n"
+ " }";
try {
ObjectMapper mapper = Zenodo.mapper;
ZenodoDeposition dep = mapper.readValue(Fixer.fixIncoming(depoisitionString), ZenodoDeposition.class);
System.out.println("The deposition: "+dep);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}