gCubeSystem
/
gFeed
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Commons module to only read opened streams

This commit is contained in:
FabioISTI 2020-06-09 11:25:58 +02:00
parent 4958fbb6e5
commit 61b2be7553
5 changed files with 103 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/Files.java
View File

@ -4,41 +4,69 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Files {
public static String readFileFromResources(String fileName) throws IOException {
ClassLoader classLoader =Files.class.getClassLoader();
URL resource = classLoader.getResource(fileName);
if (resource == null) {
throw new IllegalArgumentException("file is not found!");
}
return readStream(resource.openStream());
}
public static final String readStream(InputStream is) throws IOException {
byte[] buffer = new byte[10];
StringBuilder sb = new StringBuilder();
while (is.read(buffer) != -1) {
sb.append(new String(buffer));
buffer = new byte[10];
}
is.close();
return sb.toString();
}
private static final Logger log= LoggerFactory.getLogger(Files.class);
public static String readFileAsString(File toRead)
throws IOException{
//Classloader & resources cause problems with plugins
// public static String getProjectDir() {
// try {
// Class<?> callingClass = Class.forName(Thread.currentThread().getStackTrace()[2].getClassName());
// URL url = callingClass.getProtectionDomain().getCodeSource().getLocation();
// URI parentDir = url.toURI().resolve("..");
// return parentDir.getPath();
// } catch (ClassNotFoundException e) {
// e.printStackTrace();
// } catch (URISyntaxException e) {
// e.printStackTrace();
// }
// return "";
// }
InputStream fis = new FileInputStream(toRead);
return readStream(fis);
}
// public static String readFileFromResources(String fileName) throws IOException {
//
// ClassLoader classLoader =Files.class.getClassLoader();
//
// URL resource = classLoader.getResource(fileName);
// if (resource == null) {
// throw new IllegalArgumentException("file is not found!");
// }
// return readStream(resource.openStream());
//
// }
// public static final String readStream(InputStream is) throws IOException {
// byte[] buffer = new byte[10];
// StringBuilder sb = new StringBuilder();
// while (is.read(buffer) != -1) {
// sb.append(new String(buffer));
// buffer = new byte[10];
// }
// is.close();
//
// return sb.toString();
// }
//
// public static String readFileAsString(String toRead)
// throws IOException{
//
// String fullPath=getProjectDir()+toRead;
// log.debug("Reading "+toRead+". Full path is "+fullPath);
//
//
// InputStream fis = new FileInputStream(toRead);
// return readStream(fis);
//
// }
// public static String readFileAsString(String path, Charset encoding)

25
commons/src/main/java/org/gcube/data/publishing/gCatFeeder/utils/IOUtils.java Normal file
View File

@ -0,0 +1,25 @@
package org.gcube.data.publishing.gCatFeeder.utils;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
public class IOUtils {
public static String readStream(InputStream toRead) throws IOException {
final int bufferSize = 1024;
final char[] buffer = new char[bufferSize];
final StringBuilder out = new StringBuilder();
Reader in = new InputStreamReader(toRead, StandardCharsets.UTF_8);
int charsRead;
while((charsRead = in.read(buffer, 0, buffer.length)) > 0) {
out.append(buffer, 0, charsRead);
}
return out.toString();
}
}

23
oai-harvester/src/main/java/org/gcube/data/publishing/gFeed/collectors/oai/OAIHarvester.java
View File

@ -1,13 +1,13 @@
package org.gcube.data.publishing.gFeed.collectors.oai;
import java.nio.charset.Charset;
import java.io.InputStream;
import java.util.Collections;
import java.util.Set;
import org.gcube.data.publishing.gCatFeeder.model.CatalogueFormatData;
import org.gcube.data.publishing.gCatFeeder.model.ControllerConfiguration;
import org.gcube.data.publishing.gCatFeeder.model.EnvironmentConfiguration;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gCatFeeder.utils.IOUtils;
import org.gcube.data.publishing.gCatfeeder.collectors.CatalogueRetriever;
import org.gcube.data.publishing.gCatfeeder.collectors.CollectorPlugin;
import org.gcube.data.publishing.gCatfeeder.collectors.DataCollector;
@ -63,14 +63,25 @@ public class OAIHarvester implements CollectorPlugin<OAIRecord>{
@Override
public void init() throws Exception {
String harvestedObjectProfile=Files.readFileFromResources("profiles/HarvestedObject.xml");
String harvestedObjectProfile=IOUtils.readStream(
getClass().getClassLoader().getResourceAsStream("profiles/HarvestedObject.xml"));
if(harvestedObjectProfile==null||harvestedObjectProfile.isEmpty())
log.warn("HarvestedObject is empty");
Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile);
Set<String> profilesID=Constants.xmlProfiles.keySet();
log.info("Loaded profiles "+profilesID);
if(log.isDebugEnabled())
for(String p:profilesID)
log.debug("PROFILE "+p+" : "+Constants.xmlProfiles.get(p));
Constants.xmlProfiles.put("HarvestedObject", harvestedObjectProfile);
log.debug("Loaded profiles "+Constants.xmlProfiles.keySet());
}
@Override
public void initInScope() throws Exception {
// TODO Auto-generated method stub

4
oai-harvester/src/test/java/org/gcube/application/gfeed/oai/ParsingTests.java
View File

@ -13,6 +13,7 @@ import javax.xml.bind.Unmarshaller;
import org.gcube.data.publishing.gCatFeeder.model.InternalConversionException;
import org.gcube.data.publishing.gCatFeeder.utils.Files;
import org.gcube.data.publishing.gCatFeeder.utils.IOUtils;
import org.gcube.data.publishing.gFeed.collectors.oai.model.DCRecordMetadata;
import org.gcube.data.publishing.gFeed.collectors.oai.model.MetadataHolder;
import org.gcube.data.publishing.gFeed.collectors.oai.model.OAIMetadata;
@ -75,7 +76,8 @@ public class ParsingTests {
}
private OAI_PMH read() throws JAXBException, IOException {
String toRead=Files.readFileFromResources("resp_dc.xml");
String toRead=IOUtils.readStream(
getClass().getClassLoader().getResourceAsStream("resp_dc.xml"));
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();

2
pom.xml
View File

@ -18,7 +18,7 @@
<properties>
<distroDirectory>distro</distroDirectory>
<gitBaseUrl>https://code-repo.d4science.org/gCubeSystem</gitBaseUrl>
<gFeedSuiteVersion>1.0.3</gFeedSuiteVersion>
<gFeedSuiteVersion>1.0.3-SNAPSHOT</gFeedSuiteVersion>
</properties>
<scm>