gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes and changelog

This commit is contained in:
FabioISTI 2020-05-19 14:25:29 +02:00
parent b8f16d152f
commit b95345e49c
7 changed files with 56 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
changelog.md Normal file
View File

@ -0,0 +1,12 @@
# Changelog
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [1.0.2] - 2020-05-19
### Added
**Features**
Added new harvester for oai-pmh repository.

28
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java
View File

@ -1,10 +1,10 @@
package org.gcube.data.publishing.gCatFeeder.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Paths;
public class Files {
@ -21,14 +21,30 @@ public class Files {
}
public static String readFileAsString(File toRead, Charset encoding)
throws IOException{
FileInputStream fis = new FileInputStream(toRead);
byte[] buffer = new byte[10];
StringBuilder sb = new StringBuilder();
while (fis.read(buffer) != -1) {
sb.append(new String(buffer));
buffer = new byte[10];
}
fis.close();
public static String readFileAsString(String path, Charset encoding)
throws IOException
{
byte[] encoded = java.nio.file.Files.readAllBytes(Paths.get(path));
return new String(encoded, encoding);
return sb.toString();
}
// public static String readFileAsString(String path, Charset encoding)
// throws IOException
// {
// byte[] encoded = java.nio.file.Files.readAllBytes(Paths.get(path));
// return new String(encoded, encoding);
// }
//
public static String getName(String path) {

2
gCat-Controller/src/main/java/org/gcube/data/publishing/gCataFeeder/catalogues/gCat/GCatController.java
View File

@ -272,7 +272,7 @@ public class GCatController implements CatalogueController{
if(existsItem(item,name))
return item.update(name, toCreate);
else return item.create(toCreate);
else return item.create(toCreate,false);
}catch(WebApplicationException e) {
handleWebException(e);

5
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/Constants.java
View File

@ -1,5 +1,8 @@
package org.gcube.data.publishing.gFeed.collectors.oai;
import java.util.HashMap;
import java.util.Map;
public class Constants {
@ -7,4 +10,6 @@ public class Constants {
public static final String PLUGIN_ID="OAI_COLLECTOR";
public static final Map<String,String> xmlProfiles=new HashMap<String,String>();
}

4
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIClient.java
View File

@ -90,10 +90,10 @@ public class OAIClient {
Response resp=target.request("application/xml").get();
OAI_PMH msg=check(resp);
//No errors, thus reset attempt counter
currentAttempt=1;
if(msg.isError()) throw new OAIInteractionException(msg.getError().getCode()+ " : "+msg.getError().getMessage());
//No errors, thus reset attempt counter
currentAttempt=1;
toReturn.addAll(msg.getResponseRecords().getRecords());
log.debug("Parsed "+toReturn.size()+" records so far.");

13
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java
View File

@ -1,11 +1,13 @@
package org.gcube.data.publishing.gFeed.collectors.oai;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
@ -15,6 +17,9 @@ import org.gcube.data.publishing.gCatfeeder.collectors.model.faults.CatalogueNot
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord;
import org.gcube.data.publishing.gFeed.collectors.oai.model.ckan.GCatTransformer;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class OAIHarvester implements CollectorPlugin<OAIRecord>{
@Override
@ -57,7 +62,13 @@ public class OAIHarvester implements CollectorPlugin<OAIRecord>{
@Override
public void init() throws Exception {
// TODO Auto-generated method stub
String harvestedObjectProfile=Files.readFileAsString(
Files.getFileFromResources("profiles/HarvestedObject.xml"),
Charset.defaultCharset());
Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile);
log.debug("Loaded profiles "+Constants.xmlProfiles.keySet());
}

5
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/model/ckan/GCatTransformer.java
View File

@ -10,6 +10,7 @@ import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gCatfeeder.collectors.DataTransformer;
import org.gcube.data.publishing.gFeed.collectors.oai.Constants;
import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIRecord;
@ -126,9 +127,7 @@ public class GCatTransformer implements DataTransformer<GCatModel,OAIRecord>{
try {
if(useProfile)
toReturn.setProfile(Files.readFileAsString(
Files.getFileFromResources("profiles/HarvestedObject.xml").getAbsolutePath(),
Charset.defaultCharset())); //"Harvested Object”
toReturn.setProfile(Constants.xmlProfiles.get("HarvestedObject")); //"Harvested Object”
}catch(Throwable t) {
log.error("Unable to set profile ",t);