gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-publishing/gCat-Feeder-Suite@178595 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2019-03-14 18:07:56 +00:00
parent 20a57b80dc
commit 88dd08b31e
19 changed files with 487 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
catalogue-plugin-framework/pom.xml
View File

@ -6,7 +6,7 @@
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>catalogue-plugin-framework</artifactId>
<dependencies>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>

16
catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/CataloguePluginDescriptor.java
View File

@ -1,9 +1,17 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
@RequiredArgsConstructor
@Getter
@Setter
public class CataloguePluginDescriptor {
public CataloguePluginDescriptor() {
// TODO Auto-generated constructor stub
}
@NonNull
private String id;
}

38
catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/faults/PublicationException.java Normal file
View File

@ -0,0 +1,38 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.model.faults;
public class PublicationException extends Exception {
/**
*
*/
private static final long serialVersionUID = -2113773348645295768L;
public PublicationException() {
super();
// TODO Auto-generated constructor stub
}
public PublicationException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
// TODO Auto-generated constructor stub
}
public PublicationException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public PublicationException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public PublicationException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

36
ckan-controller-plugin/.classpath Normal file
View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

23
ckan-controller-plugin/.project Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>ckan-controller-plugin</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

6
ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs Normal file
View File

@ -0,0 +1,6 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8

5
ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs Normal file
View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8

4
ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs Normal file
View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

24
ckan-controller-plugin/pom.xml Normal file
View File

@ -0,0 +1,24 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>ckan-controller-plugin</artifactId>
<name>ckan-controller-plugin</name>
<description>controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue</description>
<dependencies>
<dependency>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>catalogue-plugin-framework</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
</dependencies>
</project>

203
ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java Normal file
View File

@ -0,0 +1,203 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.ckan;
import java.net.URL;
import java.util.Iterator;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.Response;
import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationException;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
import org.gcube.data.publishing.gCatFeeder.utils.GCubeAuthorizationFilter;
import org.gcube.data.publishing.gCatFeeder.utils.TokenUtils;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.ClientProperties;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class CKANController implements CatalogueController {
private static ObjectMapper mapper=new ObjectMapper();
private static final String PUBLISH_ITEM="catalogue-ws/rest/api/items/create";
private static final String PUBLSIH_RESOURCE="catalogue-ws/rest/api/resources/create";
private String hostname=null;
private String customToken=null;
private final int maxAttempts=3;
public CKANController(CatalogueInstanceDescriptor instance) throws ControllerInstantiationFault {
try{
log.trace("Instantiating controller towards {} ",instance);
URL url=new URL(instance.getUrl());
this.hostname=url.getHost();
this.customToken=instance.getCustomToken();
}catch(Throwable t) {
throw new ControllerInstantiationFault("Unable to instantiate CKAN controller. ",t);
}
}
/**
* Expected structure
*
* { "item" : "<serialized_item>",
* "resources" : [
* "<serialized_resource>",
* "<serialized_resource>",
* ....]
* }
*
* NB serialized resources are updated with "package_id" set as the published item id
*
*
*/
@Override
public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException {
//Get serialized format
log.debug("Publishing {} ",toPublish);
String serialized=toPublish.toCatalogueFormat();
String toResetToken=TokenUtils.getCurrentToken();
if(customToken!=null) {
log.debug("Custom token found.. Setting it..");
TokenUtils.setToken(customToken);
}
try {
log.debug("Checking serialized structure.. ");
JsonNode node=mapper.readTree(serialized);
String item=node.path("item").asText();
String itemResp=publishItem(item);
String itemUrl=getPublishedUrl(itemResp);
log.info("Published item {} ",itemUrl);
if(node.has("resources")) {
log.debug("Publishing resources..");
JsonNode resourcesNode=node.path("resources");
String itemId=getId(itemResp);
log.debug("Setting package id {} ",itemId);
Iterator<JsonNode> it=resourcesNode.iterator();
while(it.hasNext()) {
JsonNode res=it.next();
((ObjectNode)res).put("package_id",itemId);
String resResp=publishResource(res.asText());
String resUrl=getPublishedUrl(resResp);
log.info("Published resource {} ",resUrl);
}
}
return new PublishReport(true, itemUrl);
}catch(PublicationException e) {
log.error("Unable to Publish ",e);
return new PublishReport(false, e.getMessage());
}catch(Throwable t) {
throw new WrongObjectFormatException("Unable to parse Serialized object.",t);
}finally {
TokenUtils.setToken(toResetToken);
}
}
private String publishItem(String jsonItem) throws PublicationException {
String urlString="http://"+hostname+":80/"+PUBLISH_ITEM;
String respString=null;
int attemptCounter=0;
while(respString==null&&attemptCounter<maxAttempts)
try {
Response resp=getWebClient().target(urlString).request("application/json")
// .header("Authorization", ckanToken)
.post(Entity.json(jsonItem));
respString=check(resp);
}catch(PublicationException e) {
attemptCounter++;
if(attemptCounter==maxAttempts) throw new PublicationException("Unable to publish item "+jsonItem, e);
}
return respString;
}
private String publishResource(String jsonResource) throws PublicationException {
String urlString="http://"+hostname+":80/"+PUBLSIH_RESOURCE;
int attemptCounter=0;
boolean done=false;
String respString=null;
while(attemptCounter<maxAttempts&&(!done))
try {
Response resp=getWebClient().target(urlString).request("application/json")
// .header("Authorization", ckanToken)
.post(Entity.json(jsonResource));
respString=check(resp);
done=true;
}catch(PublicationException e) {
attemptCounter++;
if(attemptCounter==maxAttempts) throw new PublicationException("Unable to publish resource "+jsonResource, e);
}
return respString;
}
private static String check(Response resp) throws PublicationException {
if(resp.getStatus()<200||resp.getStatus()>=300) {
throw new PublicationException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class));
}else {
try {
String respString=resp.readEntity(String.class);
if(!mapper.readTree(respString).path("success").asBoolean())
throw new PublicationException("Error : response is "+respString);
return respString;
}catch(PublicationException e) {
throw e;
}catch(Throwable t) {
throw new PublicationException("Unable to check response ",t);
}
}
}
private static String getId(String publishResponse) {
try {
return mapper.readTree(publishResponse).path("result").path("id").asText();
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException("FAILED Parsing of "+publishResponse);
}
}
private static String getPublishedUrl(String publishResponse) {
try {
Iterator<JsonNode> iterator=mapper.readTree(publishResponse).path("result").path("extras").elements();
while(iterator.hasNext()) {
JsonNode node=iterator.next();
if(node.path("key").asText().equals("Item URL"))
return node.path("value").asText();
}
return "N/A";
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException("FAILED Parsing of "+publishResponse);
}
}
private static Client getWebClient() {
return ClientBuilder.newClient(new ClientConfig().register(GCubeAuthorizationFilter.class))
.property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true);
}
}

23
ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java Normal file
View File

@ -0,0 +1,23 @@
package org.gcube.data.publishing.gCatFeeder.catalogues.ckan;
import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor;
import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor;
public class CkanPlugin implements CataloguePlugin{
@Override
public CataloguePluginDescriptor getDescriptor() {
return new CataloguePluginDescriptor("CKAN");
}
@Override
public CKANController instantiateController(CatalogueInstanceDescriptor desc)
throws ControllerInstantiationFault {
return new CKANController(desc);
}
}

5
ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,5 @@
Manifest-Version: 1.0
Built-By: fabio
Build-Jdk: 1.8.0_201
Created-By: Maven Integration for Eclipse

7
ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties Normal file
View File

@ -0,0 +1,7 @@
#Generated by Maven Integration for Eclipse
#Thu Mar 14 17:51:18 CET 2019
version=1.0.0-SNAPSHOT
groupId=org.gcube.data-publishing.gCat-Feeder
m2e.projectName=ckan-controller-plugin
m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/ckan-controller-plugin
artifactId=ckan-controller-plugin

24
ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml Normal file
View File

@ -0,0 +1,24 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>ckan-controller-plugin</artifactId>
<name>ckan-controller-plugin</name>
<description>controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue</description>
<dependencies>
<dependency>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>catalogue-plugin-framework</artifactId>
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
</dependency>
</dependencies>
</project>

BIN
ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class Normal file
View File

Binary file not shown.

BIN
ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class Normal file
View File

Binary file not shown.

27
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/GCubeAuthorizationFilter.java Normal file
View File

@ -0,0 +1,27 @@
package org.gcube.data.publishing.gCatFeeder.utils;
import java.io.IOException;
import java.util.Collections;
import java.util.Map.Entry;
import javax.ws.rs.client.ClientRequestContext;
import javax.ws.rs.client.ClientRequestFilter;
import org.gcube.common.calls.Call;
import org.gcube.common.calls.Interceptors;
import org.gcube.common.calls.Request;
public class GCubeAuthorizationFilter implements ClientRequestFilter {
@Override
public void filter(final ClientRequestContext rc) throws IOException {
if (ContextUtils.getCurrentScope()!=null){
Request requestContext = Interceptors.executeRequestChain(new Call());
for (Entry<String, String> entry: requestContext.getHeaders()){
rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue()));
}
}
}
}

9
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java
View File

@ -1,5 +1,14 @@
package org.gcube.data.publishing.gCatFeeder.utils;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
public class TokenUtils {
public static void setToken(String token) {
SecurityTokenProvider.instance.set(token);
}
public static String getCurrentToken() {
return SecurityTokenProvider.instance.get();
}
}

56
pom.xml
View File

@ -12,14 +12,15 @@
<packaging>pom</packaging>
<name>gCat-Feeder Suite</name>
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
<modules>
<module>gCat-Feeder</module>
<module>collectors-plugin-framework</module>
<module>DataMinerAlgorithmsCrawler</module>
<module>commons</module>
<module>catalogue-plugin-framework</module>
<modules>
<module>gCat-Feeder</module>
<module>collectors-plugin-framework</module>
<module>DataMinerAlgorithmsCrawler</module>
<module>commons</module>
<module>catalogue-plugin-framework</module>
<module>ckan-controller-plugin</module>
</modules>
<dependencyManagement>
@ -33,11 +34,34 @@
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<dependencies>
<!-- GCUBE -->
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-generic-clients</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-gcube-calls</artifactId>
</dependency>
<!-- JERSEY -->
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.13</version>
@ -50,7 +74,7 @@
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencies>
</project>