gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed version

This commit is contained in:
FabioISTI 2020-05-08 14:34:12 +02:00
parent 2b976a081e
commit aaa72ee15d
16 changed files with 190 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
DataMinerAlgorithmsCrawler/pom.xml
View File

@ -4,15 +4,15 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>DataMinerAlgorithmsCrawler</artifactId> <artifactId>DataMinerAlgorithmsCrawler</artifactId>
<name>DataMinerAlgorithmsCrawler</name> <name>DataMinerAlgorithmsCrawler</name>
<description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description> <description>Plugin for gCat-Feeder for DataMiner Algorithms publishing</description>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

8
catalogue-plugin-framework/pom.xml
View File

@ -3,12 +3,12 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>catalogue-plugin-framework</artifactId> <artifactId>catalogue-plugin-framework</artifactId>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

8
collectors-plugin-framework/pom.xml
View File

@ -3,14 +3,14 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>collectors-plugin-framework</artifactId> <artifactId>collectors-plugin-framework</artifactId>
<name>collectors-plugin-framework</name> <name>collectors-plugin-framework</name>
<description>Framework for collectors implementations</description> <description>Framework for collectors implementations</description>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

8
commons/pom.xml
View File

@ -4,15 +4,15 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>commons</artifactId> <artifactId>commons</artifactId>
<name>commons</name> <name>commons</name>
<description>Common utilities for gCatFeeder logic</description> <description>Common utilities for gCatFeeder logic</description>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

8
gCat-Controller/pom.xml
View File

@ -3,14 +3,14 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>gCat-Controller</artifactId> <artifactId>gCat-Controller</artifactId>
<name>gCat Controller</name> <name>gCat Controller</name>
<description>Controller implementation for GCat Service</description> <description>Controller implementation for GCat Service</description>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

6
gCat-Controller/src/test/java/org/gcube/data/publishing/gCatFeeder/catalogues/gCat/Interactions.java
View File

@ -79,6 +79,12 @@ public class Interactions extends BaseCataloguePluginTest{
publish(getController(),"full-algorithm.json"); publish(getController(),"full-algorithm.json");
} }
@Test
public void testOAI() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, InternalConversionException, ControllerInstantiationFault {
Assume.assumeTrue(isTestInfrastructureEnabled());
publish(getController(),"oai_dc_full.json");
}
@Test(expected=WrongObjectFormatException.class) @Test(expected=WrongObjectFormatException.class)
public void empties() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault, InternalConversionException { public void empties() throws CatalogueInteractionException, WrongObjectFormatException, PublicationException, ControllerInstantiationFault, InternalConversionException {
Assume.assumeTrue(isTestInfrastructureEnabled()); Assume.assumeTrue(isTestInfrastructureEnabled());

4
gCat-Feeder/pom.xml
View File

@ -4,7 +4,7 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>gCat-Feeder</artifactId> <artifactId>gCat-Feeder</artifactId>
<name>gCat-Feeder</name> <name>gCat-Feeder</name>
@ -13,7 +13,7 @@
<properties> <properties>
<webappDirectory>${project.basedir}/src/main/webapp/WEB-INF</webappDirectory> <webappDirectory>${project.basedir}/src/main/webapp/WEB-INF</webappDirectory>
<distroDirectory>${project.basedir}/distro</distroDirectory> <distroDirectory>${project.basedir}/distro</distroDirectory>
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> </properties>
<scm> <scm>

8
oai-harvester/pom.xml
View File

@ -3,14 +3,14 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>oai-harvester</artifactId> <artifactId>oai-harvester</artifactId>
<name>oai-harvester</name> <name>oai-harvester</name>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>

32
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/DCRecordMetadata.java
View File

@ -1,5 +1,7 @@
package org.gcube.data.publishing.gFeed.collectors.oai.model; package org.gcube.data.publishing.gFeed.collectors.oai.model;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElement;
@ -23,37 +25,37 @@ public class DCRecordMetadata extends OAIMetadata{
public static final String NS="http://purl.org/dc/elements/1.1/"; public static final String NS="http://purl.org/dc/elements/1.1/";
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String contributor; private List<String> contributor;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String coverage; private List<String> coverage;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String creator; private List<String> creator;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String date; private List<String> date;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String description; private List<String> description;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String format; private List<String> format;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String identifier; private List<String> identifier;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String language; private List<String> language;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String publisher; private List<String> publisher;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String relation; private List<String> relation;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String rights; private List<String> rights;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String source; private List<String> source;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String subject; private List<String> subject;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String title; private List<String> title;
@XmlElement(namespace=NS) @XmlElement(namespace=NS)
private String type; private List<String> type;
} }

3
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatModel.java
View File

@ -37,12 +37,15 @@ public class GCatModel implements CatalogueFormatData {
private static ObjectMapper mapper=new ObjectMapper(); private static ObjectMapper mapper=new ObjectMapper();
@Setter @Setter
@Getter
private String profile=null; private String profile=null;
@Setter @Setter
@Getter
private CkanItem item; private CkanItem item;
@Setter @Setter
@Getter
private ArrayList<CkanResource> resources=new ArrayList<>(); private ArrayList<CkanResource> resources=new ArrayList<>();
@Override @Override

102
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java
View File

@ -5,6 +5,7 @@ import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Set; import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.utils.Files; import org.gcube.data.publishing.gCatFeeder.utils.Files;
@ -13,6 +14,7 @@ import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.CkanItem.CKanExtraField; import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.CkanItem.CKanExtraField;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.CkanItem.Tag;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatModel.CkanResource; import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatModel.CkanResource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -59,66 +61,65 @@ public class GCatTransformer implements DataTransformer<GCatModel,OAIRecord>{
GCatModel toReturn = new GCatModel(); GCatModel toReturn = new GCatModel();
CkanItem item=new CkanItem(); CkanItem item=new CkanItem();
//escaping name chars
item.setName(toTranslate.getHeader().getIdentifier()); String toSetName=toTranslate.getHeader().getIdentifier();
toSetName=toSetName.replaceAll("[^a-z0-9_\\\\-]", "_");
item.setName(toSetName);
OAIMetadata meta=toTranslate.getMetadata().getMetadata(); OAIMetadata meta=toTranslate.getMetadata().getMetadata();
if(meta instanceof DCRecordMetadata) { if(meta instanceof DCRecordMetadata) {
String profileID="Harvested Object"; String profileID="Harvested Object";
DCRecordMetadata dcMeta=(DCRecordMetadata) meta; DCRecordMetadata dcMeta=(DCRecordMetadata) meta;
item.setTitle(dcMeta.getTitle()); item.setTitle(getFirstOrDefault(dcMeta.getTitle(),"n.a."));
item.setNotes(dcMeta.getDescription()); item.setNotes(getFirstOrDefault(dcMeta.getDescription(),null));
item.setAuthor(dcMeta.getPublisher()); item.setAuthor(getFirstOrDefault(dcMeta.getPublisher(),"n.a."));
item.setMaintainer(dcMeta.getPublisher()); item.setMaintainer(getFirstOrDefault(dcMeta.getPublisher(),null));
item.setVersion("n.a."); item.setVersion("n.a.");
item.setPrivateFlag(false); item.setPrivateFlag(false);
item.setLicense_id("CC-BY-NC-SA-4.0"); item.setLicense_id("CC-BY-NC-SA-4.0");
if(dcMeta.getSubject()!=null&&dcMeta.getSubject().isEmpty())
for(String subject:dcMeta.getSubject())
item.getTags().add(new Tag(subject));
item.getTags().add(new Tag("OAI"));
item.getExtras().add(new CKanExtraField("system:type", profileID)); item.getExtras().add(new CKanExtraField("system:type", profileID));
if(dcMeta.getContributor()!=null)
item.getExtras().add(new CKanExtraField(profileID+":contributor", dcMeta.getContributor())); item.getExtras().addAll(fromList(dcMeta.getContributor(),profileID+":contributor"));
if(dcMeta.getCoverage()!=null) item.getExtras().addAll(fromList(dcMeta.getCoverage(),profileID+":coverage"));
item.getExtras().add(new CKanExtraField(profileID+":coverage", dcMeta.getCoverage())); item.getExtras().addAll(fromList(dcMeta.getCreator(),profileID+":creator"));
if(dcMeta.getCreator()!=null) item.getExtras().addAll(fromList(dcMeta.getDate(),profileID+":date"));
item.getExtras().add(new CKanExtraField(profileID+":creator", dcMeta.getCreator())); item.getExtras().addAll(fromList(dcMeta.getDescription(),profileID+":description"));
if(dcMeta.getDate()!=null) item.getExtras().addAll(fromList(dcMeta.getFormat(),profileID+":format"));
item.getExtras().add(new CKanExtraField(profileID+":date", dcMeta.getDate())); item.getExtras().addAll(fromList(dcMeta.getIdentifier(),profileID+":identifier"));
if(dcMeta.getDescription()!=null) item.getExtras().addAll(fromList(dcMeta.getLanguage(),profileID+":language"));
item.getExtras().add(new CKanExtraField(profileID+":description", dcMeta.getDescription())); item.getExtras().addAll(fromList(dcMeta.getPublisher(),profileID+":publisher"));
if(dcMeta.getFormat()!=null) item.getExtras().addAll(fromList(dcMeta.getRelation(),profileID+":relation"));
item.getExtras().add(new CKanExtraField(profileID+":format", dcMeta.getFormat())); item.getExtras().addAll(fromList(dcMeta.getRights(),profileID+":rights"));
if(dcMeta.getIdentifier()!=null) item.getExtras().addAll(fromList(dcMeta.getSource(),profileID+":source"));
item.getExtras().add(new CKanExtraField(profileID+":identifier", dcMeta.getIdentifier())); item.getExtras().addAll(fromList(dcMeta.getSubject(),profileID+":subject"));
if(dcMeta.getLanguage()!=null) item.getExtras().addAll(fromList(dcMeta.getTitle(),profileID+":title"));
item.getExtras().add(new CKanExtraField(profileID+":language", dcMeta.getLanguage())); item.getExtras().addAll(fromList(dcMeta.getType(),profileID+":type"));
if(dcMeta.getPublisher()!=null)
item.getExtras().add(new CKanExtraField(profileID+":publisher", dcMeta.getPublisher()));
if(dcMeta.getRelation()!=null)
item.getExtras().add(new CKanExtraField(profileID+":relation", dcMeta.getRelation()));
if(dcMeta.getRights()!=null)
item.getExtras().add(new CKanExtraField(profileID+":rights", dcMeta.getRights()));
if(dcMeta.getSource()!=null)
item.getExtras().add(new CKanExtraField(profileID+":source", dcMeta.getSource()));
if(dcMeta.getSubject()!=null)
item.getExtras().add(new CKanExtraField(profileID+":subject", dcMeta.getSubject()));
if(dcMeta.getTitle()!=null)
item.getExtras().add(new CKanExtraField(profileID+":title", dcMeta.getTitle()));
if(dcMeta.getType()!=null)
item.getExtras().add(new CKanExtraField(profileID+":type", dcMeta.getType()));
try {
URL url=new URL(dcMeta.getIdentifier()); if(dcMeta.getIdentifier()!=null) {
CkanResource res=new CkanResource("Record", url+"", url.getProtocol(), "Original record");
ArrayList<CkanResource> list=new ArrayList<>(); ArrayList<CkanResource> list=new ArrayList<>();
for(String id:dcMeta.getIdentifier()) {
try {
URL url=new URL(id);
CkanResource res=new CkanResource("Record", url+"", url.getProtocol(), "Original record");
list.add(res);
}catch(Throwable t) {
log.debug("Unable to set identifier "+id+"as resource ",t);
}
}
toReturn.setResources(list); toReturn.setResources(list);
}catch(Throwable t) {
log.debug("Unable to set identifier "+dcMeta.getIdentifier()+"as resource ",t);
} }
} }
toReturn.setItem(item); toReturn.setItem(item);
@ -126,7 +127,7 @@ public class GCatTransformer implements DataTransformer<GCatModel,OAIRecord>{
try { try {
if(useProfile) if(useProfile)
toReturn.setProfile(Files.readFileAsString( toReturn.setProfile(Files.readFileAsString(
Files.getFileFromResources("HarvesterObject.xml").getAbsolutePath(), Files.getFileFromResources("profiles/HarvestedObject.xml").getAbsolutePath(),
Charset.defaultCharset())); //"Harvested Object” Charset.defaultCharset())); //"Harvested Object”
}catch(Throwable t) { }catch(Throwable t) {
log.error("Unable to set profile ",t); log.error("Unable to set profile ",t);
@ -136,4 +137,17 @@ public class GCatTransformer implements DataTransformer<GCatModel,OAIRecord>{
} }
private static Collection<CKanExtraField> fromList(List<String> values,String name){
ArrayList<CKanExtraField> toReturn=new ArrayList<CKanExtraField>();
if(values!=null)
for(String v:values)
toReturn.add(new CKanExtraField(name,v));
return toReturn;
}
private static String getFirstOrDefault(List<String> values,String defaultValue) {
if(values!=null && !values.isEmpty()) return values.get(0);
else return defaultValue;
}
} }

30
oai-harvester/src/main/resources/profiles/HarvestedObject.xml
View File

@ -8,92 +8,92 @@
<metadataformat type="Harvested Object"> <metadataformat type="Harvested Object">
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>contributor</fieldName> <fieldName>contributor</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>coverage</fieldName> <fieldName>coverage</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>creator</fieldName> <fieldName>creator</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>date</fieldName> <fieldName>date</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>description</fieldName> <fieldName>description</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>format</fieldName> <fieldName>format</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>identifier</fieldName> <fieldName>identifier</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>language</fieldName> <fieldName>language</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>publisher</fieldName> <fieldName>publisher</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>relation</fieldName> <fieldName>relation</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>rights</fieldName> <fieldName>rights</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>source</fieldName> <fieldName>source</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>subject</fieldName> <fieldName>subject</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>title</fieldName> <fieldName>title</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
<metadatafield categoryref="Harvested Object"> <metadatafield categoryref="Harvested Object">
<fieldName>type</fieldName> <fieldName>type</fieldName>
<maxOccurs>*</maxOccurs>
<mandatory>false</mandatory> <mandatory>false</mandatory>
<dataType>String</dataType> <dataType>String</dataType>
<maxOccurs>*</maxOccurs>
</metadatafield> </metadatafield>
</metadataformat> </metadataformat>

36
oai-harvester/src/test/java/org/gcube/application/gfeed/oai/CKANTest.java
View File

@ -1,11 +1,23 @@
package org.gcube.application.gfeed.oai; package org.gcube.application.gfeed.oai;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException; import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter; import org.gcube.data.publishing.gCatFeeder.tests.TokenSetter;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.MetadataHolder;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAI_PMH;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatModel;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatTransformer;
import org.gcube.gcat.client.Item; import org.gcube.gcat.client.Item;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@ -14,9 +26,27 @@ import com.fasterxml.jackson.databind.ObjectMapper;
public class CKANTest { public class CKANTest {
public static void main (String args[]) { public static void main (String args[]) throws JAXBException {
TokenSetter.set("/pred4s/preprod/preVRE"); // TokenSetter.set("/pred4s/preprod/preVRE");
//
//
// File toRead=Files.getFileFromResources("resp_dc.xml");
//
// JAXBContext jaxbContext = JAXBContext.newInstance(OAIRecord.class,
// MetadataHolder.class,
// OAIMetadata.class,
// DCRecordMetadata.class,
// OAI_PMH.class);
//
// Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
//
// OAI_PMH response = (OAI_PMH) jaxbUnmarshaller.unmarshal(toRead);
//
// for(GCatModel gcat: new GCatTransformer().transform(response.getResponseRecords().getRecords())) {
// if(gcat.)
// }
//
} }

56
oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java
View File

@ -4,19 +4,21 @@ import static org.junit.Assert.assertTrue;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.nio.file.Paths;
import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException; import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller; import javax.xml.bind.Unmarshaller;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.MetadataHolder; import org.gcube.data.publishing.gFeed.collectors.oai.model.MetadataHolder;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAI_PMH; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAI_PMH;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatModel;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatTransformer;
import org.junit.Assert; import org.junit.Assert;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
@ -40,12 +42,8 @@ public class ParsingTests {
@Test @Test
public void parseDC() throws JAXBException { public void parseDC() throws JAXBException {
File toRead=getFileFromResources("resp_dc.xml");
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); OAI_PMH response=read();
OAI_PMH response = (OAI_PMH) jaxbUnmarshaller.unmarshal(toRead);
Assert.assertNotNull(response); Assert.assertNotNull(response);
Assert.assertNotNull(response.getResponseRecords().getRecords()); Assert.assertNotNull(response.getResponseRecords().getRecords());
@ -61,29 +59,27 @@ public class ParsingTests {
} }
@Test @Test
public void marshal() throws JAXBException { public void marshal() throws JAXBException, InternalConversionException, IOException {
// jaxbContext.createMarshaller().marshal(new OAI_PMH("dummy"),System.out); OAI_PMH resp=read();
File outF=File.createTempFile("json", ".json");
PrintWriter out = new PrintWriter(outF);
for(GCatModel gcat: new GCatTransformer().transform(resp.getResponseRecords().getRecords())) {
Assert.assertNotNull(gcat.getResources());
Assert.assertFalse(gcat.getResources().isEmpty());
out.println(gcat.toCatalogueFormat());
}
out.flush();
out.close();
System.out.println("Output at "+outF.getAbsolutePath());
} }
private OAI_PMH read() throws JAXBException {
public static File getFileFromResources(String fileName) { File toRead=Files.getFileFromResources("resp_dc.xml");
ClassLoader classLoader =ParsingTests.class.getClassLoader(); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
URL resource = classLoader.getResource(fileName); return (OAI_PMH) jaxbUnmarshaller.unmarshal(toRead);
if (resource == null) {
throw new IllegalArgumentException("file is not found!");
} else {
return new File(resource.getFile());
}
}
public static String readFileAsString(String path, Charset encoding)
throws IOException
{
byte[] encoded = java.nio.file.Files.readAllBytes(Paths.get(path));
return new String(encoded, encoding);
} }
} }

3
pom.xml
View File

@ -8,7 +8,7 @@
</parent> </parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>gCat-Feeder Suite</name> <name>gCat-Feeder Suite</name>
<description>gCat-Feeder Suite of components : service, plugin framework, plugins</description> <description>gCat-Feeder Suite of components : service, plugin framework, plugins</description>
@ -17,6 +17,7 @@
<properties> <properties>
<distroDirectory>distro</distroDirectory> <distroDirectory>distro</distroDirectory>
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
<gFeedSuiteVersion>1.0.2-SNAPSHOT</gFeedSuiteVersion>
</properties> </properties>
<scm> <scm>

8
test-commons/pom.xml
View File

@ -3,14 +3,14 @@
<parent> <parent>
<groupId>org.gcube.data-publishing.gCat-Feeder</groupId> <groupId>org.gcube.data-publishing.gCat-Feeder</groupId>
<artifactId>gCat-Feeder-Suite</artifactId> <artifactId>gCat-Feeder-Suite</artifactId>
<version>1.0.1</version> <version>${gFeedSuiteVersion}</version>
</parent> </parent>
<artifactId>test-commons</artifactId> <artifactId>test-commons</artifactId>
<name>test commons</name> <name>test commons</name>
<description>Utility classes for test purposes</description> <description>Utility classes for test purposes</description>
<properties> <!-- <properties> -->
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> <!-- <gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl> -->
</properties> <!-- </properties> -->
<scm> <scm>
<connection>scm:git:${gitBaseUrl}/gFeed</connection> <connection>scm:git:${gitBaseUrl}/gFeed</connection>