diff --git a/catalogue-plugin-framework/pom.xml b/catalogue-plugin-framework/pom.xml index 3f9219a..be278e1 100644 --- a/catalogue-plugin-framework/pom.xml +++ b/catalogue-plugin-framework/pom.xml @@ -6,7 +6,7 @@ 1.0.0-SNAPSHOT catalogue-plugin-framework - + org.projectlombok lombok diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/CataloguePluginDescriptor.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/CataloguePluginDescriptor.java index af0f9bf..0b225d5 100644 --- a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/CataloguePluginDescriptor.java +++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/CataloguePluginDescriptor.java @@ -1,9 +1,17 @@ package org.gcube.data.publishing.gCatFeeder.catalogues.model; +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +@RequiredArgsConstructor +@Getter +@Setter public class CataloguePluginDescriptor { - public CataloguePluginDescriptor() { - // TODO Auto-generated constructor stub - } - + @NonNull + private String id; + + } diff --git a/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/faults/PublicationException.java b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/faults/PublicationException.java new file mode 100644 index 0000000..d7e17bb --- /dev/null +++ b/catalogue-plugin-framework/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/model/faults/PublicationException.java @@ -0,0 +1,38 @@ +package org.gcube.data.publishing.gCatFeeder.catalogues.model.faults; + +public class PublicationException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -2113773348645295768L; + + public PublicationException() { + super(); + // TODO Auto-generated constructor stub + } + + public PublicationException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + // TODO Auto-generated constructor stub + } + + public PublicationException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public PublicationException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public PublicationException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } + + + +} diff --git a/ckan-controller-plugin/.classpath b/ckan-controller-plugin/.classpath new file mode 100644 index 0000000..fae1a2b --- /dev/null +++ b/ckan-controller-plugin/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ckan-controller-plugin/.project b/ckan-controller-plugin/.project new file mode 100644 index 0000000..29e74b7 --- /dev/null +++ b/ckan-controller-plugin/.project @@ -0,0 +1,23 @@ + + + ckan-controller-plugin + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs b/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/ckan-controller-plugin/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs b/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..714351a --- /dev/null +++ b/ckan-controller-plugin/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs b/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/ckan-controller-plugin/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/ckan-controller-plugin/pom.xml b/ckan-controller-plugin/pom.xml new file mode 100644 index 0000000..b69b1db --- /dev/null +++ b/ckan-controller-plugin/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + ckan-controller-plugin + ckan-controller-plugin + controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue + + + + + org.gcube.data-publishing.gCat-Feeder + catalogue-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + + + \ No newline at end of file diff --git a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java new file mode 100644 index 0000000..0585358 --- /dev/null +++ b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.java @@ -0,0 +1,203 @@ +package org.gcube.data.publishing.gCatFeeder.catalogues.ckan; + +import java.net.URL; +import java.util.Iterator; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Response; + +import org.gcube.data.publishing.gCatFeeder.catalogues.CatalogueController; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.PublishReport; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.PublicationException; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.WrongObjectFormatException; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; +import org.gcube.data.publishing.gCatFeeder.utils.GCubeAuthorizationFilter; +import org.gcube.data.publishing.gCatFeeder.utils.TokenUtils; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CKANController implements CatalogueController { + + private static ObjectMapper mapper=new ObjectMapper(); + + + + private static final String PUBLISH_ITEM="catalogue-ws/rest/api/items/create"; + private static final String PUBLSIH_RESOURCE="catalogue-ws/rest/api/resources/create"; + + private String hostname=null; + private String customToken=null; + private final int maxAttempts=3; + + public CKANController(CatalogueInstanceDescriptor instance) throws ControllerInstantiationFault { + try{ + log.trace("Instantiating controller towards {} ",instance); + URL url=new URL(instance.getUrl()); + this.hostname=url.getHost(); + this.customToken=instance.getCustomToken(); + }catch(Throwable t) { + throw new ControllerInstantiationFault("Unable to instantiate CKAN controller. ",t); + } + } + + + /** + * Expected structure + * + * { "item" : "", + * "resources" : [ + * "", + * "", + * ....] + * } + * + * NB serialized resources are updated with "package_id" set as the published item id + * + * + */ + @Override + public PublishReport publishItem(CatalogueFormatData toPublish) throws WrongObjectFormatException { + //Get serialized format + log.debug("Publishing {} ",toPublish); + String serialized=toPublish.toCatalogueFormat(); + String toResetToken=TokenUtils.getCurrentToken(); + + if(customToken!=null) { + log.debug("Custom token found.. Setting it.."); + TokenUtils.setToken(customToken); + } + + + try { + log.debug("Checking serialized structure.. "); + JsonNode node=mapper.readTree(serialized); + String item=node.path("item").asText(); + String itemResp=publishItem(item); + String itemUrl=getPublishedUrl(itemResp); + log.info("Published item {} ",itemUrl); + if(node.has("resources")) { + log.debug("Publishing resources.."); + JsonNode resourcesNode=node.path("resources"); + String itemId=getId(itemResp); + log.debug("Setting package id {} ",itemId); + Iterator it=resourcesNode.iterator(); + while(it.hasNext()) { + JsonNode res=it.next(); + ((ObjectNode)res).put("package_id",itemId); + String resResp=publishResource(res.asText()); + String resUrl=getPublishedUrl(resResp); + log.info("Published resource {} ",resUrl); + } + } + + return new PublishReport(true, itemUrl); + + }catch(PublicationException e) { + log.error("Unable to Publish ",e); + return new PublishReport(false, e.getMessage()); + }catch(Throwable t) { + throw new WrongObjectFormatException("Unable to parse Serialized object.",t); + }finally { + TokenUtils.setToken(toResetToken); + } + } + + + + private String publishItem(String jsonItem) throws PublicationException { + String urlString="http://"+hostname+":80/"+PUBLISH_ITEM; + String respString=null; + int attemptCounter=0; + while(respString==null&&attemptCounter=300) { + throw new PublicationException("RESP STATUS IS "+resp.getStatus()+". Message : "+resp.readEntity(String.class)); + }else { + try { + String respString=resp.readEntity(String.class); + if(!mapper.readTree(respString).path("success").asBoolean()) + throw new PublicationException("Error : response is "+respString); + return respString; + }catch(PublicationException e) { + throw e; + }catch(Throwable t) { + throw new PublicationException("Unable to check response ",t); + } + } + } + + private static String getId(String publishResponse) { + try { + return mapper.readTree(publishResponse).path("result").path("id").asText(); + } catch (Throwable t) { + t.printStackTrace(); + throw new RuntimeException("FAILED Parsing of "+publishResponse); + } + } + + private static String getPublishedUrl(String publishResponse) { + try { + Iterator iterator=mapper.readTree(publishResponse).path("result").path("extras").elements(); + while(iterator.hasNext()) { + JsonNode node=iterator.next(); + if(node.path("key").asText().equals("Item URL")) + return node.path("value").asText(); + } + return "N/A"; + } catch (Throwable t) { + t.printStackTrace(); + throw new RuntimeException("FAILED Parsing of "+publishResponse); + } + } + + private static Client getWebClient() { + return ClientBuilder.newClient(new ClientConfig().register(GCubeAuthorizationFilter.class)) + .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true); + } + + +} diff --git a/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java new file mode 100644 index 0000000..bb1b958 --- /dev/null +++ b/ckan-controller-plugin/src/main/java/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.java @@ -0,0 +1,23 @@ +package org.gcube.data.publishing.gCatFeeder.catalogues.ckan; + +import org.gcube.data.publishing.gCatFeeder.catalogues.CataloguePlugin; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.CataloguePluginDescriptor; +import org.gcube.data.publishing.gCatFeeder.catalogues.model.faults.ControllerInstantiationFault; +import org.gcube.data.publishing.gCatFeeder.model.CatalogueInstanceDescriptor; + +public class CkanPlugin implements CataloguePlugin{ + + @Override + public CataloguePluginDescriptor getDescriptor() { + return new CataloguePluginDescriptor("CKAN"); + } + + @Override + public CKANController instantiateController(CatalogueInstanceDescriptor desc) + throws ControllerInstantiationFault { + return new CKANController(desc); + } + + + +} diff --git a/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF b/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF new file mode 100644 index 0000000..e2601c5 --- /dev/null +++ b/ckan-controller-plugin/target/classes/META-INF/MANIFEST.MF @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +Built-By: fabio +Build-Jdk: 1.8.0_201 +Created-By: Maven Integration for Eclipse + diff --git a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties new file mode 100644 index 0000000..43cea8e --- /dev/null +++ b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.properties @@ -0,0 +1,7 @@ +#Generated by Maven Integration for Eclipse +#Thu Mar 14 17:51:18 CET 2019 +version=1.0.0-SNAPSHOT +groupId=org.gcube.data-publishing.gCat-Feeder +m2e.projectName=ckan-controller-plugin +m2e.projectLocation=/home/fabio/workspaces/DEV/gCat-Feeder-Suite/ckan-controller-plugin +artifactId=ckan-controller-plugin diff --git a/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml new file mode 100644 index 0000000..b69b1db --- /dev/null +++ b/ckan-controller-plugin/target/classes/META-INF/maven/org.gcube.data-publishing.gCat-Feeder/ckan-controller-plugin/pom.xml @@ -0,0 +1,24 @@ + + 4.0.0 + + org.gcube.data-publishing.gCat-Feeder + gCat-Feeder-Suite + 1.0.0-SNAPSHOT + + ckan-controller-plugin + ckan-controller-plugin + controller plugin for gCat-Feeder aimed to publish metadata to CKAN catalogue + + + + + org.gcube.data-publishing.gCat-Feeder + catalogue-plugin-framework + [1.0.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + + + + + \ No newline at end of file diff --git a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class new file mode 100644 index 0000000..7d78483 Binary files /dev/null and b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CKANController.class differ diff --git a/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class new file mode 100644 index 0000000..c3e6317 Binary files /dev/null and b/ckan-controller-plugin/target/classes/org/gcube/data/publishing/gCatFeeder/catalogues/ckan/CkanPlugin.class differ diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/GCubeAuthorizationFilter.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/GCubeAuthorizationFilter.java new file mode 100644 index 0000000..d9868a4 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/GCubeAuthorizationFilter.java @@ -0,0 +1,27 @@ +package org.gcube.data.publishing.gCatFeeder.utils; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map.Entry; + +import javax.ws.rs.client.ClientRequestContext; +import javax.ws.rs.client.ClientRequestFilter; + +import org.gcube.common.calls.Call; +import org.gcube.common.calls.Interceptors; +import org.gcube.common.calls.Request; + +public class GCubeAuthorizationFilter implements ClientRequestFilter { + + + @Override + public void filter(final ClientRequestContext rc) throws IOException { + if (ContextUtils.getCurrentScope()!=null){ + Request requestContext = Interceptors.executeRequestChain(new Call()); + + for (Entry entry: requestContext.getHeaders()){ + rc.getHeaders().put(entry.getKey(), Collections.singletonList((Object)entry.getValue())); + } + } + } +} diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java index fef0c82..32e4e7c 100644 --- a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/TokenUtils.java @@ -1,5 +1,14 @@ package org.gcube.data.publishing.gCatFeeder.utils; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; + public class TokenUtils { + public static void setToken(String token) { + SecurityTokenProvider.instance.set(token); + } + + public static String getCurrentToken() { + return SecurityTokenProvider.instance.get(); + } } diff --git a/pom.xml b/pom.xml index 624a86f..2c07fb9 100644 --- a/pom.xml +++ b/pom.xml @@ -12,14 +12,15 @@ pom gCat-Feeder Suite gCat-Feeder Suite of components : service, plugin framework, plugins - - - - gCat-Feeder - collectors-plugin-framework - DataMinerAlgorithmsCrawler - commons - catalogue-plugin-framework + + + + gCat-Feeder + collectors-plugin-framework + DataMinerAlgorithmsCrawler + commons + catalogue-plugin-framework + ckan-controller-plugin @@ -33,11 +34,34 @@ - - - - - + + + + + + + org.gcube.core + common-generic-clients + + + + org.gcube.core + common-gcube-calls + + + + + org.glassfish.jersey.core + jersey-client + + + + + org.glassfish.jersey.media + jersey-media-json-jackson + + + ch.qos.logback logback-classic 1.0.13 @@ -50,7 +74,7 @@ 4.12 test - - - + + + \ No newline at end of file