D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Irish oaipmh exporter #443

Merged
claudio.atzori merged 7 commits from irish-oaipmh-exporter into beta 2024-06-05 10:55:09 +02:00
Conversation 0 Commits 7 Files Changed 10 +13 -25
2 changed files with 13 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c9a327bc50 - Show all commits

14
dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java
View File

@ -4,6 +4,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Optional;
@ -145,15 +146,14 @@ public class IrishOaiExporterJob {
protected static byte[] gzip(final String str) {
if (StringUtils.isBlank(str)) { return null; }
try {
final ByteArrayOutputStream obj = new ByteArrayOutputStream();
final GZIPOutputStream gzip = new GZIPOutputStream(obj);
gzip.write(str.getBytes("UTF-8"));
gzip.flush();
gzip.close();
return obj.toByteArray();
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
try (final GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
IOUtils.write(str.getBytes(Charset.defaultCharset()), gzip);
}
return baos.toByteArray();
} catch (final IOException e) {
throw new RuntimeException("error in gzip", e);
}
}
}

24
dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java
View File

@ -6,10 +6,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.IOUtils;
@ -57,7 +56,7 @@ class IrishOaiExporterJobTest {
final byte[] bytes = IrishOaiExporterJob.gzip(message);
assertNotNull(bytes);
assertTrue(bytes.length > 0);
assertEquals(message, decompress(bytes));
assertEquals(message, gunzip(bytes));
}
@Test
@ -66,22 +65,11 @@ class IrishOaiExporterJobTest {
assertNull(IrishOaiExporterJob.gzip(null));
}
private static String decompress(final byte[] compressed) {
final StringBuilder outStr = new StringBuilder();
public static String gunzip(final byte[] compressed) {
if ((compressed == null) || (compressed.length == 0)) { return null; }
try {
if (isCompressed(compressed)) {
final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed));
final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gis, "UTF-8"));
String line;
while ((line = bufferedReader.readLine()) != null) {
outStr.append(line);
}
} else {
outStr.append(compressed);
}
return outStr.toString();
if (!isCompressed(compressed)) { return new String(compressed); }
try (final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed))) {
return IOUtils.toString(gis, Charset.defaultCharset());
} catch (final IOException e) {
throw new RuntimeException("error in gunzip", e);
}