gCubeSystem
/
ckan2zenodo-library
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request '1.0.0-SNAPSHOT' (#2) from 1.0.0-SNAPSHOT into master 

Reviewed-on: #2
This commit is contained in:
Fabio Sinibaldi 2021-04-19 12:19:56 +02:00
parent a2495dbe64 c5b03675e6
commit d7b05a2b57
5 changed files with 185 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CHANGELOG.md
View File

@ -1,6 +1,10 @@
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
# Changelog for org.gcube.data.publishing.ckan2zenodo-library
## [v1.0.1] 2021-04-7
-Integration with gCat 2.0.0 [#20751](https://support.d4science.org/issues/20751)
## [v1.0.0] 2021-02-18
- Improved mapping language : target record
@ -14,15 +18,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
## [v0.0.2] 2020-06-30
### Enhancements
- Default Ckan2Zenodo translation to comply with mandatory Zenodo fields (https://support.d4science.org/issues/19489)
- Ckan2Zenodo library to provide means to apply default translation only (https://support.d4science.org/issues/19490)
- Support to "split" on source values (https://support.d4science.org/issues/19534)
- Support to "append" to target elements (https://support.d4science.org/issues/19535)
- Multiple Date Format parsing (https://support.d4science.org/issues/19540)
- Default Ckan2Zenodo translation to comply with mandatory Zenodo fields [19489](https://support.d4science.org/issues/19489)
- Ckan2Zenodo library to provide means to apply default translation only [19490](https://support.d4science.org/issues/19490)
- Support to "split" on source values [19534](https://support.d4science.org/issues/19534)
- Support to "append" to target elements [19534](https://support.d4science.org/issues/19535)
- Multiple Date Format parsing [19540](https://support.d4science.org/issues/19540)
- Automatically set item URL if missing.
### Fixes
- Default filter resource is applied in class Ckan2ZenodoImpl.java (https://support.d4science.org/issues/19528)
- Default filter resource is applied in class Ckan2ZenodoImpl.java [19528](https://support.d4science.org/issues/19528)
- Fixed error message for missing profile.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

18
pom.xml
View File

@ -8,7 +8,7 @@
</parent>
<groupId>org.gcube.data.publishing</groupId>
<artifactId>ckan2zenodo-library</artifactId>
<version>1.0.0</version>
<version>1.0.1</version>
<name>CKAN 2 Zenodo Library</name>
<description>Library to publish d4science CKAN items into Zenodo</description>
@ -31,7 +31,7 @@
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>gcube-bom</artifactId>
<version>2.0.0</version>
<version>2.0.1</version>
<type>pom</type>
<scope>import</scope>
</dependency>
@ -39,19 +39,13 @@
</dependencyManagement>
<dependencies>
<!-- <dependency> -->
<!-- <groupId>org.gcube.data-catalogue</groupId> -->
<!-- <artifactId>gcat-client</artifactId> -->
<!-- <version>[2.0.0-SNAPSHOT,3.0.0)</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.gcube.data-publishing</groupId>
<dependency>
<groupId>org.gcube.data-catalogue</groupId>
<artifactId>gcat-client</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0)</version>
<version>[2.0.0-SNAPSHOT,3.0.0)</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-encryption</artifactId>

44
src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java
View File

@ -1,10 +1,12 @@
package org.gcube.data.publishing.ckan2zenodo.model;
import java.io.IOException;
import java.util.List;
import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
@ -46,6 +48,11 @@ public class CkanItemDescriptor {
private static final String NAME="$.name";
private static final String ZENODO_DOI="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')]";
private static final String ZENODO_DOI_VALUE="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')].value";
private static final String ORGANIZATION="$.organization";
private static final String RESOURCES="$.resources";
private static final String RESOURCES_COUNT="$.num_resources";
private static final String ID="$.id";
@NonNull
@Getter
@ -168,8 +175,41 @@ public class CkanItemDescriptor {
}
}
public CkanItemDescriptor setItemUrl(String toSet) {
getDocument().set(ITEM_URL,toSet);
public CkanResource getResources() throws JsonProcessingException, IOException{
return getDocument().read(RESOURCES,CkanResource.class);
// ArrayList<CkanResource> toReturn=new ArrayList<>();
//
// for(Object o: mapper.readerFor(CkanResource.class).readValues().readAll())
// toReturn.add((CkanResource) o);
// return toReturn;
}
public CkanItemDescriptor cleanResources() {
getDocument().set(RESOURCES,new Object[0]);
getDocument().set(RESOURCES_COUNT,0);
content=getDocument().jsonString();
return this;
}
public CkanItemDescriptor removeOrganization() {
getDocument().set(ORGANIZATION,null);
content=getDocument().jsonString();
return this;
}
public CkanItemDescriptor removeId() {
getDocument().set(ID,null);
content=getDocument().jsonString();
return this;
}
public CkanItemDescriptor setItemUrl(String toSet) {
getDocument().set(ITEM_URL,toSet);
content=getDocument().jsonString();
return this;
}
}

124
src/test/java/org/gcube/tests/OneHitTest.java
View File

@ -1,6 +1,7 @@
package org.gcube.tests;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
@ -8,6 +9,7 @@ import java.util.concurrent.Future;
import org.gcube.data.publishing.ckan2zenodo.Ckan2Zenodo;
import org.gcube.data.publishing.ckan2zenodo.Ckan2ZenodoImpl;
import org.gcube.data.publishing.ckan2zenodo.clients.GCat;
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
@ -16,48 +18,118 @@ import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
import org.gcube.gcat.client.Item;
public class OneHitTest {
public static void main(String[] args) throws GcatException, InvalidItemException, ZenodoException, ConfigurationException, TransformationException, InterruptedException, ExecutionException, MalformedURLException {
// String scope="/gcube/devsec";
String scope="/pred4s/preprod/preVRE";
TokenSetter.set(scope);
Ckan2Zenodo client=new Ckan2ZenodoImpl();
String toPublishItemName="a_sample_deliverable_4_zenodo";
// String scope="/gcube/devsec";
// String scope="/pred4s/preprod/preVRE";
// String scope="/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab";
// TokenSetter.set(scope);
//
//
// // String toPublishItemName="a_sample_deliverable_4_zenodo";
// String toPublishItemName="blue_cloud_demonstrator_users_handbook_v1";
// Get the item representation
// Import & TEST
String sourceContext="/d4science.research-infrastructures.eu/D4OS/Blue-CloudProject";
String targetContext="/pred4s/preprod/preVRE";
Boolean publish=true;
String[] items= {
"blue_cloud_demonstrator_users_handbook_v1"
};
// IMPORT
importItemsByProfile(sourceContext, targetContext, items).forEach((CkanItemDescriptor d)->{
try {
testFullCycle(targetContext, d.getName(), publish);
} catch (GcatException | InvalidItemException | ZenodoException | ConfigurationException
| TransformationException | InterruptedException | ExecutionException | MalformedURLException e) {
throw new RuntimeException(e);
}
});;
// testFullCycle(sourceContext, items[0], false);
}
public static void testFullCycle(String context,String toPublishItemName, boolean publish) throws GcatException, InvalidItemException, ZenodoException, ConfigurationException, TransformationException, InterruptedException, ExecutionException, MalformedURLException {
TokenSetter.set(context);
Ckan2Zenodo client=new Ckan2ZenodoImpl();
CkanItemDescriptor item=client.read(toPublishItemName);
System.out.println("Item PROFILE is : "+item.getProfile());
//Get a preview of the deposition to be published
ZenodoDeposition preview=client.translate(item);
System.out.println("DEPOSITION SOULD BE "+preview);
//Filter resources according to VRE policies
List<CkanResource> toFilter=client.filterResources(item);
//Eventually update values
preview.getMetadata().setAccess_conditions("Ask me");
//Actually publish to zenodo :
// Step 1 : metadata
System.out.println("Create/update metadata..");
preview=client.updatedMetadata(preview);
System.out.println("Publish files..");
//Step 2 : publish Resources
Future<ZenodoDeposition> future_Dep=client.uploadFiles(Collections.singleton(toFilter.get(0)), preview);
preview=future_Dep.get();
System.out.println("Publishing.. ");
//Finalize
System.out.println("DONE : "+client.publish(preview, item));
if(publish) {
System.out.println("!!!! PUBLISHING.. ");
//Finalize
try{
Thread.sleep(5000);
}catch(InterruptedException e) {}
System.out.println("Create/update metadata..");
preview=client.updatedMetadata(preview);
System.out.println("Publish files..");
//Step 2 : publish Resources
Future<ZenodoDeposition> future_Dep=client.uploadFiles(Collections.singleton(toFilter.get(0)), preview);
preview=future_Dep.get();
System.out.println("DONE : "+client.publish(preview, item));
}
}
public static ArrayList<CkanItemDescriptor> importItemsByProfile(String sourceContext,String targetContext,String... items) throws MalformedURLException, GcatException {
System.out.println("Importing from "+sourceContext);
TokenSetter.set(sourceContext);
ArrayList<CkanItemDescriptor> toReturn=new ArrayList<CkanItemDescriptor>();
for(String s:items)
toReturn.add(GCat.getByID(s));
System.out.println("Pushing in "+targetContext);
TokenSetter.set(targetContext);
for(CkanItemDescriptor d:toReturn) {
try{
d.removeOrganization();
d.removeId();
// List<CkanResource> res=
d.getResources();
d.cleanResources();
new Item().create(d.getContent());
}catch(Exception e) {
System.err.println("Cannot publish "+d.getContent());
e.printStackTrace(System.err);
GCat.updateItem(d);
}
}
return toReturn;
}
}

58
src/test/resources/blue_cloud_deliverable.xml
View File

@ -24,43 +24,43 @@
<targetElement>upload_type</targetElement>
</mapping>
<!-- <mapping> -->
<!-- <source> -->
<!-- <value type="jsonPath">$.extras[?(@.key=='Deliverable Author')].value</value> -->
<!-- </source> -->
<!-- <targetPath append="false" type="array">$.metadata.creators</targetPath> -->
<!-- <targetElement>name</targetElement> -->
<!-- </mapping> -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Deliverable Author')].value</value>
</source>
<targetPath append="false" type="array">$.metadata.creators</targetPath>
<targetElement>name</targetElement>
</mapping>
<!-- Contributor -> {name : ... , type : 'other'} -->
<!-- <mapping> -->
<!-- <source> -->
<!-- <value type="jsonPath">$.extras[?(@.key=='Deliverable Contributor')].value</value> -->
<!-- </source> -->
<!-- <targetPath append="false" type="array">$.metadata.contributors</targetPath> -->
<!-- <targetElement>name</targetElement> -->
<!-- <targetElement constant="Other">type</targetElement> -->
<!-- </mapping> -->
<mapping>
<source>
<value type="jsonPath">$.extras[?(@.key=='Deliverable Contributor')].value</value>
</source>
<targetPath append="false" type="array">$.metadata.contributors</targetPath>
<targetElement>name</targetElement>
<targetElement constant="Other">type</targetElement>
</mapping>
<!-- <mapping> -->
<!-- <source> -->
<!-- <value type="constant">862409</value> -->
<!-- </source> -->
<!-- <targetPath type="array">$.metadata.grants</targetPath> -->
<!-- <targetElement>id</targetElement> -->
<!-- </mapping> -->
<mapping>
<source>
<value type="constant">862409</value>
</source>
<targetPath type="array">$.metadata.grants</targetPath>
<targetElement>id</targetElement>
</mapping>
<!-- <mapping> -->
<!-- <source> -->
<!-- <value type="constant">blue-cloud</value> -->
<!-- </source> -->
<!-- <targetPath type="array">$.metadata.communities</targetPath> -->
<!-- <targetElement>identifier</targetElement> -->
<!-- </mapping> -->
<mapping>
<source>
<value type="constant">bluecloud</value>
</source>
<targetPath type="array">$.metadata.communities</targetPath>
<targetElement>identifier</targetElement>
</mapping>
</mappings>