diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..8dfcdf5 --- /dev/null +++ b/changelog.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.2] - 2020-05-19 + +### Added + +**Features** + +Added new harvester for oai-pmh repository. diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java index 40139ac..3e866b9 100644 --- a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java @@ -1,10 +1,10 @@ package org.gcube.data.publishing.gCatFeeder.utils; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.net.URL; import java.nio.charset.Charset; -import java.nio.file.Paths; public class Files { @@ -21,14 +21,30 @@ public class Files { } + public static String readFileAsString(File toRead, Charset encoding) + throws IOException{ + + FileInputStream fis = new FileInputStream(toRead); + byte[] buffer = new byte[10]; + StringBuilder sb = new StringBuilder(); + while (fis.read(buffer) != -1) { + sb.append(new String(buffer)); + buffer = new byte[10]; + } + fis.close(); - public static String readFileAsString(String path, Charset encoding) - throws IOException - { - byte[] encoded = java.nio.file.Files.readAllBytes(Paths.get(path)); - return new String(encoded, encoding); + return sb.toString(); + } + +// public static String readFileAsString(String path, Charset encoding) +// throws IOException +// { +// byte[] encoded = java.nio.file.Files.readAllBytes(Paths.get(path)); +// return new String(encoded, encoding); +// } +// public static String getName(String path) { diff --git a/gCat-Controller/src/main/java/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.java b/gCat-Controller/src/main/java/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.java index 77ec9ec..36ff8b0 100644 --- a/gCat-Controller/src/main/java/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.java +++ b/gCat-Controller/src/main/java/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.java @@ -272,7 +272,7 @@ public class GCatController implements CatalogueController{ if(existsItem(item,name)) return item.update(name, toCreate); - else return item.create(toCreate); + else return item.create(toCreate,false); }catch(WebApplicationException e) { handleWebException(e); diff --git a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/Constants.java b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/Constants.java index 63004c0..f2cac78 100644 --- a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/Constants.java +++ b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/Constants.java @@ -1,5 +1,8 @@ package org.gcube.data.publishing.gFeed.collectors.oai; +import java.util.HashMap; +import java.util.Map; + public class Constants { @@ -7,4 +10,6 @@ public class Constants { public static final String PLUGIN_ID="OAI_COLLECTOR"; + + public static final Map xmlProfiles=new HashMap(); } diff --git a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java index c557130..29193fd 100644 --- a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java +++ b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java @@ -90,10 +90,10 @@ public class OAIClient { Response resp=target.request("application/xml").get(); OAI_PMH msg=check(resp); - //No errors, thus reset attempt counter - currentAttempt=1; if(msg.isError()) throw new OAIInteractionException(msg.getError().getCode()+ " : "+msg.getError().getMessage()); + //No errors, thus reset attempt counter + currentAttempt=1; toReturn.addAll(msg.getResponseRecords().getRecords()); log.debug("Parsed "+toReturn.size()+" records so far."); diff --git a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java index 2eeedca..49bc1ea 100644 --- a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java +++ b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java @@ -1,11 +1,13 @@ package org.gcube.data.publishing.gFeed.collectors.oai; +import java.nio.charset.Charset; import java.util.Collections; import java.util.Set; import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration; import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration; +import org.gcube.data.publishing.gCatFeeder.utils.Files; import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; @@ -15,6 +17,9 @@ import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNot import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord; import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatTransformer; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class OAIHarvester implements CollectorPlugin{ @Override @@ -57,7 +62,13 @@ public class OAIHarvester implements CollectorPlugin{ @Override public void init() throws Exception { - // TODO Auto-generated method stub + + String harvestedObjectProfile=Files.readFileAsString( + Files.getFileFromResources("profiles/HarvestedObject.xml"), + Charset.defaultCharset()); + + Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile); + log.debug("Loaded profiles "+Constants.xmlProfiles.keySet()); } diff --git a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java index b1ea038..b8319c4 100644 --- a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java +++ b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java @@ -10,6 +10,7 @@ import java.util.Set; import org.gcube.data.publishing.gCatFeeder.utils.Files; import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer; +import org.gcube.data.publishing.gFeed.collectors.oai.Constants; import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord; @@ -126,9 +127,7 @@ public class GCatTransformer implements DataTransformer{ try { if(useProfile) - toReturn.setProfile(Files.readFileAsString( - Files.getFileFromResources("profiles/HarvestedObject.xml").getAbsolutePath(), - Charset.defaultCharset())); //"Harvested Object” + toReturn.setProfile(Constants.xmlProfiles.get("HarvestedObject")); //"Harvested Object” }catch(Throwable t) { log.error("Unable to set profile ",t);