From 61b2be7553d00a7be9824124bfae3afc2d3465e1 Mon Sep 17 00:00:00 2001 From: FabioISTI Date: Tue, 9 Jun 2020 11:25:58 +0200 Subject: [PATCH] Commons module to only read opened streams --- .../publishing/gCatFeeder/utils/Files.java | 86 ++++++++++++------- .../publishing/gCatFeeder/utils/IOUtils.java | 25 ++++++ .../gFeed/collectors/oai/OAIHarvester.java | 23 +++-- .../application/gfeed/oai/ParsingTests.java | 4 +- pom.xml | 2 +- 5 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/IOUtils.java diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java index 1b77442..bbe19e4 100644 --- a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java @@ -4,41 +4,69 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + public class Files { - public static String readFileFromResources(String fileName) throws IOException { - - ClassLoader classLoader =Files.class.getClassLoader(); - - URL resource = classLoader.getResource(fileName); - if (resource == null) { - throw new IllegalArgumentException("file is not found!"); - } - return readStream(resource.openStream()); - - } - - public static final String readStream(InputStream is) throws IOException { - byte[] buffer = new byte[10]; - StringBuilder sb = new StringBuilder(); - while (is.read(buffer) != -1) { - sb.append(new String(buffer)); - buffer = new byte[10]; - } - is.close(); - - return sb.toString(); - } + private static final Logger log= LoggerFactory.getLogger(Files.class); - public static String readFileAsString(File toRead) - throws IOException{ + //Classloader & resources cause problems with plugins +// public static String getProjectDir() { +// try { +// Class callingClass = Class.forName(Thread.currentThread().getStackTrace()[2].getClassName()); +// URL url = callingClass.getProtectionDomain().getCodeSource().getLocation(); +// URI parentDir = url.toURI().resolve(".."); +// return parentDir.getPath(); +// } catch (ClassNotFoundException e) { +// e.printStackTrace(); +// } catch (URISyntaxException e) { +// e.printStackTrace(); +// } +// return ""; +// } - InputStream fis = new FileInputStream(toRead); - return readStream(fis); - - } + +// public static String readFileFromResources(String fileName) throws IOException { +// +// ClassLoader classLoader =Files.class.getClassLoader(); +// +// URL resource = classLoader.getResource(fileName); +// if (resource == null) { +// throw new IllegalArgumentException("file is not found!"); +// } +// return readStream(resource.openStream()); +// +// } + +// public static final String readStream(InputStream is) throws IOException { +// byte[] buffer = new byte[10]; +// StringBuilder sb = new StringBuilder(); +// while (is.read(buffer) != -1) { +// sb.append(new String(buffer)); +// buffer = new byte[10]; +// } +// is.close(); +// +// return sb.toString(); +// } +// +// public static String readFileAsString(String toRead) +// throws IOException{ +// +// String fullPath=getProjectDir()+toRead; +// log.debug("Reading "+toRead+". Full path is "+fullPath); +// +// +// InputStream fis = new FileInputStream(toRead); +// return readStream(fis); +// +// } // public static String readFileAsString(String path, Charset encoding) diff --git a/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/IOUtils.java b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/IOUtils.java new file mode 100644 index 0000000..fa96707 --- /dev/null +++ b/commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/IOUtils.java @@ -0,0 +1,25 @@ +package org.gcube.data.publishing.gCatFeeder.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; + +public class IOUtils { + + public static String readStream(InputStream toRead) throws IOException { + + final int bufferSize = 1024; + final char[] buffer = new char[bufferSize]; + final StringBuilder out = new StringBuilder(); + Reader in = new InputStreamReader(toRead, StandardCharsets.UTF_8); + int charsRead; + while((charsRead = in.read(buffer, 0, buffer.length)) > 0) { + out.append(buffer, 0, charsRead); + } + return out.toString(); + + } + +} diff --git a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java index c36ce02..d7daf6c 100644 --- a/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java +++ b/oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java @@ -1,13 +1,13 @@ package org.gcube.data.publishing.gFeed.collectors.oai; -import java.nio.charset.Charset; +import java.io.InputStream; import java.util.Collections; import java.util.Set; import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData; import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration; import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration; -import org.gcube.data.publishing.gCatFeeder.utils.Files; +import org.gcube.data.publishing.gCatFeeder.utils.IOUtils; import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever; import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin; import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector; @@ -63,14 +63,25 @@ public class OAIHarvester implements CollectorPlugin{ @Override public void init() throws Exception { - String harvestedObjectProfile=Files.readFileFromResources("profiles/HarvestedObject.xml"); - + String harvestedObjectProfile=IOUtils.readStream( + getClass().getClassLoader().getResourceAsStream("profiles/HarvestedObject.xml")); + + + + if(harvestedObjectProfile==null||harvestedObjectProfile.isEmpty()) + log.warn("HarvestedObject is empty"); + + Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile); + Set profilesID=Constants.xmlProfiles.keySet(); + log.info("Loaded profiles "+profilesID); + if(log.isDebugEnabled()) + for(String p:profilesID) + log.debug("PROFILE "+p+" : "+Constants.xmlProfiles.get(p)); - Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile); - log.debug("Loaded profiles "+Constants.xmlProfiles.keySet()); } + @Override public void initInScope() throws Exception { // TODO Auto-generated method stub diff --git a/oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java b/oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java index 1c24c88..d4ff8e4 100644 --- a/oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java +++ b/oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java @@ -13,6 +13,7 @@ import javax.xml.bind.Unmarshaller; import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException; import org.gcube.data.publishing.gCatFeeder.utils.Files; +import org.gcube.data.publishing.gCatFeeder.utils.IOUtils; import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata; import org.gcube.data.publishing.gFeed.collectors.oai.model.MetadataHolder; import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata; @@ -75,7 +76,8 @@ public class ParsingTests { } private OAI_PMH read() throws JAXBException, IOException { - String toRead=Files.readFileFromResources("resp_dc.xml"); + String toRead=IOUtils.readStream( + getClass().getClassLoader().getResourceAsStream("resp_dc.xml")); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); diff --git a/pom.xml b/pom.xml index c169616..43ea874 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ distro https://code-repo.d4science.org/gCubeSystem - 1.0.3 + 1.0.3-SNAPSHOT