Merge pull request '1.0.0-SNAPSHOT' (#2) from 1.0.0-SNAPSHOT into master
Reviewed-on: #2
This commit is contained in:
commit
d7b05a2b57
16
CHANGELOG.md
16
CHANGELOG.md
|
@ -1,6 +1,10 @@
|
||||||
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
# Changelog for org.gcube.data.publishing.ckan2zenodo-library
|
# Changelog for org.gcube.data.publishing.ckan2zenodo-library
|
||||||
|
|
||||||
|
## [v1.0.1] 2021-04-7
|
||||||
|
|
||||||
|
-Integration with gCat 2.0.0 [#20751](https://support.d4science.org/issues/20751)
|
||||||
|
|
||||||
## [v1.0.0] 2021-02-18
|
## [v1.0.0] 2021-02-18
|
||||||
|
|
||||||
- Improved mapping language : target record
|
- Improved mapping language : target record
|
||||||
|
@ -14,15 +18,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
||||||
## [v0.0.2] 2020-06-30
|
## [v0.0.2] 2020-06-30
|
||||||
|
|
||||||
### Enhancements
|
### Enhancements
|
||||||
- Default Ckan2Zenodo translation to comply with mandatory Zenodo fields (https://support.d4science.org/issues/19489)
|
- Default Ckan2Zenodo translation to comply with mandatory Zenodo fields [19489](https://support.d4science.org/issues/19489)
|
||||||
- Ckan2Zenodo library to provide means to apply default translation only (https://support.d4science.org/issues/19490)
|
- Ckan2Zenodo library to provide means to apply default translation only [19490](https://support.d4science.org/issues/19490)
|
||||||
- Support to "split" on source values (https://support.d4science.org/issues/19534)
|
- Support to "split" on source values [19534](https://support.d4science.org/issues/19534)
|
||||||
- Support to "append" to target elements (https://support.d4science.org/issues/19535)
|
- Support to "append" to target elements [19534](https://support.d4science.org/issues/19535)
|
||||||
- Multiple Date Format parsing (https://support.d4science.org/issues/19540)
|
- Multiple Date Format parsing [19540](https://support.d4science.org/issues/19540)
|
||||||
- Automatically set item URL if missing.
|
- Automatically set item URL if missing.
|
||||||
|
|
||||||
### Fixes
|
### Fixes
|
||||||
- Default filter resource is applied in class Ckan2ZenodoImpl.java (https://support.d4science.org/issues/19528)
|
- Default filter resource is applied in class Ckan2ZenodoImpl.java [19528](https://support.d4science.org/issues/19528)
|
||||||
- Fixed error message for missing profile.
|
- Fixed error message for missing profile.
|
||||||
|
|
||||||
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
18
pom.xml
18
pom.xml
|
@ -8,7 +8,7 @@
|
||||||
</parent>
|
</parent>
|
||||||
<groupId>org.gcube.data.publishing</groupId>
|
<groupId>org.gcube.data.publishing</groupId>
|
||||||
<artifactId>ckan2zenodo-library</artifactId>
|
<artifactId>ckan2zenodo-library</artifactId>
|
||||||
<version>1.0.0</version>
|
<version>1.0.1</version>
|
||||||
<name>CKAN 2 Zenodo Library</name>
|
<name>CKAN 2 Zenodo Library</name>
|
||||||
<description>Library to publish d4science CKAN items into Zenodo</description>
|
<description>Library to publish d4science CKAN items into Zenodo</description>
|
||||||
|
|
||||||
|
@ -31,7 +31,7 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.distribution</groupId>
|
<groupId>org.gcube.distribution</groupId>
|
||||||
<artifactId>gcube-bom</artifactId>
|
<artifactId>gcube-bom</artifactId>
|
||||||
<version>2.0.0</version>
|
<version>2.0.1</version>
|
||||||
<type>pom</type>
|
<type>pom</type>
|
||||||
<scope>import</scope>
|
<scope>import</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
@ -39,19 +39,13 @@
|
||||||
</dependencyManagement>
|
</dependencyManagement>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<!-- <dependency> -->
|
<dependency>
|
||||||
<!-- <groupId>org.gcube.data-catalogue</groupId> -->
|
<groupId>org.gcube.data-catalogue</groupId>
|
||||||
<!-- <artifactId>gcat-client</artifactId> -->
|
|
||||||
<!-- <version>[2.0.0-SNAPSHOT,3.0.0)</version> -->
|
|
||||||
<!-- </dependency> -->
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.gcube.data-publishing</groupId>
|
|
||||||
<artifactId>gcat-client</artifactId>
|
<artifactId>gcat-client</artifactId>
|
||||||
<version>[1.0.0-SNAPSHOT,2.0.0)</version>
|
<version>[2.0.0-SNAPSHOT,3.0.0)</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.gcube.core</groupId>
|
<groupId>org.gcube.core</groupId>
|
||||||
<artifactId>common-encryption</artifactId>
|
<artifactId>common-encryption</artifactId>
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
package org.gcube.data.publishing.ckan2zenodo.model;
|
package org.gcube.data.publishing.ckan2zenodo.model;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
|
import org.gcube.data.publishing.ckan2zenodo.commons.Parsing;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
|
import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
import com.jayway.jsonpath.JsonPath;
|
import com.jayway.jsonpath.JsonPath;
|
||||||
|
@ -46,6 +48,11 @@ public class CkanItemDescriptor {
|
||||||
private static final String NAME="$.name";
|
private static final String NAME="$.name";
|
||||||
private static final String ZENODO_DOI="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')]";
|
private static final String ZENODO_DOI="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')]";
|
||||||
private static final String ZENODO_DOI_VALUE="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')].value";
|
private static final String ZENODO_DOI_VALUE="$.extras[?(@.key=='relatedIdentifier:Zenodo.DOI')].value";
|
||||||
|
private static final String ORGANIZATION="$.organization";
|
||||||
|
private static final String RESOURCES="$.resources";
|
||||||
|
private static final String RESOURCES_COUNT="$.num_resources";
|
||||||
|
private static final String ID="$.id";
|
||||||
|
|
||||||
|
|
||||||
@NonNull
|
@NonNull
|
||||||
@Getter
|
@Getter
|
||||||
|
@ -168,8 +175,41 @@ public class CkanItemDescriptor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public CkanItemDescriptor setItemUrl(String toSet) {
|
|
||||||
getDocument().set(ITEM_URL,toSet);
|
public CkanResource getResources() throws JsonProcessingException, IOException{
|
||||||
|
return getDocument().read(RESOURCES,CkanResource.class);
|
||||||
|
// ArrayList<CkanResource> toReturn=new ArrayList<>();
|
||||||
|
//
|
||||||
|
// for(Object o: mapper.readerFor(CkanResource.class).readValues().readAll())
|
||||||
|
// toReturn.add((CkanResource) o);
|
||||||
|
// return toReturn;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CkanItemDescriptor cleanResources() {
|
||||||
|
getDocument().set(RESOURCES,new Object[0]);
|
||||||
|
getDocument().set(RESOURCES_COUNT,0);
|
||||||
|
content=getDocument().jsonString();
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public CkanItemDescriptor removeOrganization() {
|
||||||
|
getDocument().set(ORGANIZATION,null);
|
||||||
|
content=getDocument().jsonString();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CkanItemDescriptor removeId() {
|
||||||
|
getDocument().set(ID,null);
|
||||||
|
content=getDocument().jsonString();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CkanItemDescriptor setItemUrl(String toSet) {
|
||||||
|
getDocument().set(ITEM_URL,toSet);
|
||||||
|
content=getDocument().jsonString();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package org.gcube.tests;
|
package org.gcube.tests;
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
|
@ -8,6 +9,7 @@ import java.util.concurrent.Future;
|
||||||
|
|
||||||
import org.gcube.data.publishing.ckan2zenodo.Ckan2Zenodo;
|
import org.gcube.data.publishing.ckan2zenodo.Ckan2Zenodo;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.Ckan2ZenodoImpl;
|
import org.gcube.data.publishing.ckan2zenodo.Ckan2ZenodoImpl;
|
||||||
|
import org.gcube.data.publishing.ckan2zenodo.clients.GCat;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
|
import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
|
import org.gcube.data.publishing.ckan2zenodo.model.CkanResource;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
|
import org.gcube.data.publishing.ckan2zenodo.model.faults.ConfigurationException;
|
||||||
|
@ -16,48 +18,118 @@ import org.gcube.data.publishing.ckan2zenodo.model.faults.InvalidItemException;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
|
import org.gcube.data.publishing.ckan2zenodo.model.faults.TransformationException;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
|
import org.gcube.data.publishing.ckan2zenodo.model.faults.ZenodoException;
|
||||||
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
|
import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition;
|
||||||
|
import org.gcube.gcat.client.Item;
|
||||||
|
|
||||||
public class OneHitTest {
|
public class OneHitTest {
|
||||||
|
|
||||||
|
|
||||||
public static void main(String[] args) throws GcatException, InvalidItemException, ZenodoException, ConfigurationException, TransformationException, InterruptedException, ExecutionException, MalformedURLException {
|
public static void main(String[] args) throws GcatException, InvalidItemException, ZenodoException, ConfigurationException, TransformationException, InterruptedException, ExecutionException, MalformedURLException {
|
||||||
// String scope="/gcube/devsec";
|
// String scope="/gcube/devsec";
|
||||||
String scope="/pred4s/preprod/preVRE";
|
// String scope="/pred4s/preprod/preVRE";
|
||||||
TokenSetter.set(scope);
|
// String scope="/d4science.research-infrastructures.eu/D4OS/Blue-CloudLab";
|
||||||
|
// TokenSetter.set(scope);
|
||||||
Ckan2Zenodo client=new Ckan2ZenodoImpl();
|
//
|
||||||
String toPublishItemName="a_sample_deliverable_4_zenodo";
|
//
|
||||||
|
// // String toPublishItemName="a_sample_deliverable_4_zenodo";
|
||||||
|
// String toPublishItemName="blue_cloud_demonstrator_users_handbook_v1";
|
||||||
|
|
||||||
// Get the item representation
|
// Get the item representation
|
||||||
|
|
||||||
|
// Import & TEST
|
||||||
|
|
||||||
|
String sourceContext="/d4science.research-infrastructures.eu/D4OS/Blue-CloudProject";
|
||||||
|
String targetContext="/pred4s/preprod/preVRE";
|
||||||
|
|
||||||
|
Boolean publish=true;
|
||||||
|
String[] items= {
|
||||||
|
"blue_cloud_demonstrator_users_handbook_v1"
|
||||||
|
};
|
||||||
|
|
||||||
|
// IMPORT
|
||||||
|
importItemsByProfile(sourceContext, targetContext, items).forEach((CkanItemDescriptor d)->{
|
||||||
|
try {
|
||||||
|
testFullCycle(targetContext, d.getName(), publish);
|
||||||
|
} catch (GcatException | InvalidItemException | ZenodoException | ConfigurationException
|
||||||
|
| TransformationException | InterruptedException | ExecutionException | MalformedURLException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
});;
|
||||||
|
|
||||||
|
// testFullCycle(sourceContext, items[0], false);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public static void testFullCycle(String context,String toPublishItemName, boolean publish) throws GcatException, InvalidItemException, ZenodoException, ConfigurationException, TransformationException, InterruptedException, ExecutionException, MalformedURLException {
|
||||||
|
TokenSetter.set(context);
|
||||||
|
Ckan2Zenodo client=new Ckan2ZenodoImpl();
|
||||||
CkanItemDescriptor item=client.read(toPublishItemName);
|
CkanItemDescriptor item=client.read(toPublishItemName);
|
||||||
|
|
||||||
System.out.println("Item PROFILE is : "+item.getProfile());
|
System.out.println("Item PROFILE is : "+item.getProfile());
|
||||||
//Get a preview of the deposition to be published
|
//Get a preview of the deposition to be published
|
||||||
ZenodoDeposition preview=client.translate(item);
|
ZenodoDeposition preview=client.translate(item);
|
||||||
|
|
||||||
|
System.out.println("DEPOSITION SOULD BE "+preview);
|
||||||
//Filter resources according to VRE policies
|
//Filter resources according to VRE policies
|
||||||
List<CkanResource> toFilter=client.filterResources(item);
|
List<CkanResource> toFilter=client.filterResources(item);
|
||||||
|
|
||||||
//Eventually update values
|
//Eventually update values
|
||||||
preview.getMetadata().setAccess_conditions("Ask me");
|
preview.getMetadata().setAccess_conditions("Ask me");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//Actually publish to zenodo :
|
//Actually publish to zenodo :
|
||||||
// Step 1 : metadata
|
// Step 1 : metadata
|
||||||
|
if(publish) {
|
||||||
System.out.println("Create/update metadata..");
|
System.out.println("!!!! PUBLISHING.. ");
|
||||||
preview=client.updatedMetadata(preview);
|
//Finalize
|
||||||
|
try{
|
||||||
System.out.println("Publish files..");
|
Thread.sleep(5000);
|
||||||
//Step 2 : publish Resources
|
}catch(InterruptedException e) {}
|
||||||
Future<ZenodoDeposition> future_Dep=client.uploadFiles(Collections.singleton(toFilter.get(0)), preview);
|
|
||||||
preview=future_Dep.get();
|
|
||||||
|
|
||||||
System.out.println("Publishing.. ");
|
System.out.println("Create/update metadata..");
|
||||||
//Finalize
|
preview=client.updatedMetadata(preview);
|
||||||
|
|
||||||
System.out.println("DONE : "+client.publish(preview, item));
|
System.out.println("Publish files..");
|
||||||
|
//Step 2 : publish Resources
|
||||||
|
Future<ZenodoDeposition> future_Dep=client.uploadFiles(Collections.singleton(toFilter.get(0)), preview);
|
||||||
|
preview=future_Dep.get();
|
||||||
|
|
||||||
|
System.out.println("DONE : "+client.publish(preview, item));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static ArrayList<CkanItemDescriptor> importItemsByProfile(String sourceContext,String targetContext,String... items) throws MalformedURLException, GcatException {
|
||||||
|
System.out.println("Importing from "+sourceContext);
|
||||||
|
TokenSetter.set(sourceContext);
|
||||||
|
ArrayList<CkanItemDescriptor> toReturn=new ArrayList<CkanItemDescriptor>();
|
||||||
|
for(String s:items)
|
||||||
|
toReturn.add(GCat.getByID(s));
|
||||||
|
|
||||||
|
System.out.println("Pushing in "+targetContext);
|
||||||
|
TokenSetter.set(targetContext);
|
||||||
|
|
||||||
|
|
||||||
|
for(CkanItemDescriptor d:toReturn) {
|
||||||
|
try{
|
||||||
|
d.removeOrganization();
|
||||||
|
d.removeId();
|
||||||
|
// List<CkanResource> res=
|
||||||
|
d.getResources();
|
||||||
|
d.cleanResources();
|
||||||
|
|
||||||
|
|
||||||
|
new Item().create(d.getContent());
|
||||||
|
}catch(Exception e) {
|
||||||
|
System.err.println("Cannot publish "+d.getContent());
|
||||||
|
e.printStackTrace(System.err);
|
||||||
|
GCat.updateItem(d);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return toReturn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,43 +24,43 @@
|
||||||
<targetElement>upload_type</targetElement>
|
<targetElement>upload_type</targetElement>
|
||||||
</mapping>
|
</mapping>
|
||||||
|
|
||||||
<!-- <mapping> -->
|
<mapping>
|
||||||
<!-- <source> -->
|
<source>
|
||||||
<!-- <value type="jsonPath">$.extras[?(@.key=='Deliverable Author')].value</value> -->
|
<value type="jsonPath">$.extras[?(@.key=='Deliverable Author')].value</value>
|
||||||
<!-- </source> -->
|
</source>
|
||||||
<!-- <targetPath append="false" type="array">$.metadata.creators</targetPath> -->
|
<targetPath append="false" type="array">$.metadata.creators</targetPath>
|
||||||
<!-- <targetElement>name</targetElement> -->
|
<targetElement>name</targetElement>
|
||||||
<!-- </mapping> -->
|
</mapping>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Contributor -> {name : ... , type : 'other'} -->
|
<!-- Contributor -> {name : ... , type : 'other'} -->
|
||||||
<!-- <mapping> -->
|
<mapping>
|
||||||
<!-- <source> -->
|
<source>
|
||||||
<!-- <value type="jsonPath">$.extras[?(@.key=='Deliverable Contributor')].value</value> -->
|
<value type="jsonPath">$.extras[?(@.key=='Deliverable Contributor')].value</value>
|
||||||
<!-- </source> -->
|
</source>
|
||||||
<!-- <targetPath append="false" type="array">$.metadata.contributors</targetPath> -->
|
<targetPath append="false" type="array">$.metadata.contributors</targetPath>
|
||||||
<!-- <targetElement>name</targetElement> -->
|
<targetElement>name</targetElement>
|
||||||
<!-- <targetElement constant="Other">type</targetElement> -->
|
<targetElement constant="Other">type</targetElement>
|
||||||
<!-- </mapping> -->
|
</mapping>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<!-- <mapping> -->
|
<mapping>
|
||||||
<!-- <source> -->
|
<source>
|
||||||
<!-- <value type="constant">862409</value> -->
|
<value type="constant">862409</value>
|
||||||
<!-- </source> -->
|
</source>
|
||||||
<!-- <targetPath type="array">$.metadata.grants</targetPath> -->
|
<targetPath type="array">$.metadata.grants</targetPath>
|
||||||
<!-- <targetElement>id</targetElement> -->
|
<targetElement>id</targetElement>
|
||||||
<!-- </mapping> -->
|
</mapping>
|
||||||
|
|
||||||
<!-- <mapping> -->
|
<mapping>
|
||||||
<!-- <source> -->
|
<source>
|
||||||
<!-- <value type="constant">blue-cloud</value> -->
|
<value type="constant">bluecloud</value>
|
||||||
<!-- </source> -->
|
</source>
|
||||||
<!-- <targetPath type="array">$.metadata.communities</targetPath> -->
|
<targetPath type="array">$.metadata.communities</targetPath>
|
||||||
<!-- <targetElement>identifier</targetElement> -->
|
<targetElement>identifier</targetElement>
|
||||||
<!-- </mapping> -->
|
</mapping>
|
||||||
</mappings>
|
</mappings>
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue